{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.74, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_max", "source_row": {"Best score (across scorers)": "0.74", "Country": "United States of America", "Model version": "gpt-6-astra_max", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-10-13T13:42:38.000Z", "Training compute (FLOP)": "9.4e+26", "id": "KaEir82ukWPm2Fq4FanY2M", "mean_score": "0.74", "stderr": "0.0440844002276808"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_max", "source_row": {"Best score (across scorers)": "0.8", "Country": "United States of America", "Model version": "gpt-6-astra_max", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-09-10T20:22:39.000Z", "Training compute (FLOP)": "9.4e+26", "id": "BSzNeBCFhFUnZswSqHtAed", "mean_score": "0.8", "stderr": "0.04928872620664329"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-30", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.84, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_max", "source_row": {"Best score (across scorers)": "0.84", "Country": "United States of America", "Model version": "gpt-6-astra_max", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-08-30T14:57:43.000Z", "Training compute (FLOP)": "9.4e+26", "id": "FA7pM9hKdkgTXmGHptcno7", "mean_score": "0.84", "stderr": "0.036845294917747066"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-30", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.976, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_xhigh", "source_row": {"Best score (across scorers)": "0.976", "Country": "United States of America", "Model version": "gpt-6-astra_xhigh", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-08-30T02:22:12.000Z", "Training compute (FLOP)": "9.4e+26", "id": "AfKJ2DGMhkjZmeFZxGoY3w", "mean_score": "0.976", "stderr": "0.024"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-30", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9368421052631579, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_max", "source_row": {"Best score (across scorers)": "0.9368421052631579", "Country": "United States of America", "Model version": "gpt-6-astra_max", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-08-30T14:57:43.000Z", "Training compute (FLOP)": "9.4e+26", "id": "csK2fe5T8p2R7WgoJUu6bZ", "mean_score": "0.9368421052631579", "stderr": "0.014434038295343787"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gpt-6-astra_max", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-08-30T14:57:43.000Z", "Training compute (FLOP)": "9.4e+26", "id": "L7kW8yxFKCjVXhxNnt3VTq", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9577020202020202, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_max", "source_row": {"Best score (across scorers)": "0.9577020202020202", "Country": "United States of America", "Model version": "gpt-6-astra_max", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-08-30T14:57:43.000Z", "Training compute (FLOP)": "9.4e+26", "id": "3h8STJpG2VzY9LiBUzwotv", "mean_score": "0.9577020202020202", "stderr": "0.013696812097020311"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9472503333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6-astra-20260903", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-astra-20260903", "rows": [{"accuracy": "0.945286", "provider_name": "Amazon Bedrock", "total_questions": "396"}, {"accuracy": "0.9431713315363881", "provider_name": "Azure", "total_questions": "742"}, {"accuracy": "0.9472503333333333", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-30", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.756, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-astra_max", "source_row": {"Best score (across scorers)": "0.756", "Country": "United States of America", "Model version": "gpt-6-astra_max", "Organization": "OpenAI", "Release date": "2026-09-03", "Started at": "2026-08-30T14:57:43.000Z", "Training compute (FLOP)": "9.4e+26", "id": "PppwnX7EjXx2SeTcFN8Ujn", "mean_score": "0.756", "stderr": "0.013588548437881386"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-6-astra", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6766665000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6-astra-20260903", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-astra-20260903", "rows": [{"accuracy": "0.6733334999999999", "provider_name": "Amazon Bedrock", "total_questions": "100"}, {"accuracy": "0.6966665000000001", "provider_name": "Azure", "total_questions": "100"}, {"accuracy": "0.6766665000000001", "provider_name": "OpenAI", "total_questions": "100"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.68, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "0.68", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-10-13T12:40:12.000Z", "id": "KkGEgrjYDnofMoUJBCykcH", "mean_score": "0.68", "stderr": "0.04688261722621507"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-29", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "0.8", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-09-29T18:29:32.000Z", "id": "R4vjsSfDrDo6ECQWGTUdWe", "mean_score": "0.8", "stderr": "0.04928872620664329"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-29", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "0.8", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-09-29T22:00:34.000Z", "id": "fQzkNRV69sKaJX7o9KKneQ", "mean_score": "0.8", "stderr": "0.04020151261036849"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-29", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-09-29T22:00:34.000Z", "id": "HgAUm5zxrWbkfYUdeJMGzL", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-29", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9368421052631579, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "0.9368421052631579", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-09-29T22:00:34.000Z", "id": "aBoQLaqnXsLPrsbN8wYgUp", "mean_score": "0.9368421052631579", "stderr": "0.014434038295343787"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-29", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-09-29T22:00:34.000Z", "id": "Di7y6sES4PGQbXNkXYhdqM", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-29", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9539141414141414, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "0.9539141414141414", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-09-29T22:00:34.000Z", "id": "LspviLnbeZ82WCngWp6rwR", "mean_score": "0.9539141414141414", "stderr": "0.013780748450364692"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.944444, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6.1-sol-20260929", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6.1-sol-20260929", "rows": [{"accuracy": "0.947811", "provider_name": "Amazon Bedrock", "total_questions": "198"}, {"accuracy": "0.9382713333333333", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.944444", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-29", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.739, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6.1-sol_max", "source_row": {"Best score (across scorers)": "0.739", "Country": "United States of America", "Model version": "gpt-6.1-sol_max", "Organization": "OpenAI", "Release date": "2026-09-29", "Started at": "2026-09-29T22:00:34.000Z", "id": "dfT5VD7cQug8XW6GQHyJUF", "mean_score": "0.739", "stderr": "0.013895037677965185"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-6-1-sol", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6822223333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6.1-sol-20260929", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6.1-sol-20260929", "rows": [{"accuracy": "0.726667", "provider_name": "Amazon Bedrock", "total_questions": "50"}, {"accuracy": "0.6833333333333333", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.6822223333333333", "provider_name": "OpenAI", "total_questions": "150"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.67, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "0.67", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-10-13T13:01:31.000Z", "id": "F2tuCm5VmNDAaYYh6qYdRy", "mean_score": "0.67", "stderr": "0.04725815626252609"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-22", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8333333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "0.8333333333333334", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-09-22T16:51:00.000Z", "id": "4QrkRM7KEDCQvg4wvfhz7N", "mean_score": "0.8333333333333334", "stderr": "0.047040472216649226"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-29", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.71, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "0.71", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-09-29T17:34:54.000Z", "id": "fREi6AAmnfqUWkDqUhLLtw", "mean_score": "0.71", "stderr": "0.04560480215720687"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-22", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.95, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "0.95", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-09-22T18:27:39.000Z", "id": "9qKrTxPhpfW985gieKU56x", "mean_score": "0.95", "stderr": "0.041175470771058854"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-22", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9122807017543859, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "0.9122807017543859", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-09-22T18:27:39.000Z", "id": "LPxhvBNEn7S3xkRZUgcvc9", "mean_score": "0.9122807017543859", "stderr": "0.016786209858442095"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-22", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-09-22T18:27:39.000Z", "id": "LSfona7P3jCAnZFXJ2YA7F", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-22", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9059343434343434, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "0.9059343434343434", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-09-22T18:27:39.000Z", "id": "6gmbahPSxHf9xVhAYqpnhi", "mean_score": "0.9059343434343434", "stderr": "0.019434397190192135"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9020199999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-opus-5.5-20260921", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "rows": [{"accuracy": "0.8992142666666664", "provider_name": "Amazon Bedrock", "total_questions": "2970"}, {"accuracy": "0.9020199999999999", "provider_name": "Anthropic", "total_questions": "990"}, {"accuracy": "0.8986529999999999", "provider_name": "Azure", "total_questions": "1980"}, {"accuracy": "0.8942760000000001", "provider_name": "Claude Platform on AWS", "total_questions": "990"}, {"accuracy": "0.8998876666666668", "provider_name": "Google", "total_questions": "2970"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-22", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.722, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5-5_max", "source_row": {"Best score (across scorers)": "0.722", "Country": "United States of America", "Model version": "claude-opus-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-22", "Started at": "2026-09-22T18:27:39.000Z", "id": "CgLfiGcNwMPVzxr8GdNUAK", "mean_score": "0.722", "stderr": "0.014174516461485273"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-opus-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.756, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-opus-5.5-20260921", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5.5-20260921", "rows": [{"accuracy": "0.7519999333333334", "provider_name": "Amazon Bedrock", "total_questions": "750"}, {"accuracy": "0.756", "provider_name": "Anthropic", "total_questions": "250"}, {"accuracy": "0.7506668", "provider_name": "Azure", "total_questions": "500"}, {"accuracy": "0.7466666000000001", "provider_name": "Claude Platform on AWS", "total_questions": "250"}, {"accuracy": "0.7431110666666667", "provider_name": "Google", "total_questions": "750"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.59, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "0.59", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-10-13T13:00:38.000Z", "id": "kXU32N6tFkQLx5ei6nRJFS", "mean_score": "0.59", "stderr": "0.04943110704237103"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-29", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.75, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "0.75", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-09-29T15:17:08.000Z", "id": "5xgoV6NXxdDqscnFNpP2vv", "mean_score": "0.75", "stderr": "0.05510647116121488"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-29", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.65, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "0.65", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-09-29T15:15:14.000Z", "id": "3PVdaSEYFJKgYUgHnF4DBu", "mean_score": "0.65", "stderr": "0.04793724854411022"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-29", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8048780487804879, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "0.8048780487804879", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-09-29T15:15:14.000Z", "id": "Xo7ecFEMqPJ4HicvJTvWZZ", "mean_score": "0.8048780487804879", "stderr": "0.06265967111543964"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-29", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.887719298245614, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "0.887719298245614", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-09-29T15:15:14.000Z", "id": "4tZWwV9tqEgsCaKeSuSoG4", "mean_score": "0.887719298245614", "stderr": "0.018734030360524248"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-29", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-09-29T15:15:14.000Z", "id": "bSEaZLPehiNVDQL7HP9EJS", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-29", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9558080808080808, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "0.9558080808080808", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-09-29T15:15:14.000Z", "id": "QPk8jbJvo4986sbWtdte6J", "mean_score": "0.9558080808080808", "stderr": "0.013710320714521202"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.92929275, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-sonnet-5.5-20260928", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "rows": [{"accuracy": "0.9280299999999999", "provider_name": "Amazon Bedrock", "total_questions": "792"}, {"accuracy": "0.92929275", "provider_name": "Anthropic", "total_questions": "792"}, {"accuracy": "0.9166664999999998", "provider_name": "Azure", "total_questions": "1584"}, {"accuracy": "0.9276092499999999", "provider_name": "Claude Platform on AWS", "total_questions": "792"}, {"accuracy": "0.9305554166666667", "provider_name": "Google", "total_questions": "2376"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-29", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.465, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5-5_max", "source_row": {"Best score (across scorers)": "0.465", "Country": "United States of America", "Model version": "claude-sonnet-5-5_max", "Organization": "Anthropic", "Release date": "2026-09-28", "Started at": "2026-09-29T15:15:14.000Z", "id": "GtJ5yd3ZPQ7HVcm6Q9ssQz", "mean_score": "0.465", "stderr": "0.015780495050030086"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-sonnet-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.775, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-sonnet-5.5-20260928", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5.5-20260928", "rows": [{"accuracy": "0.7883334999999999", "provider_name": "Amazon Bedrock", "total_questions": "200"}, {"accuracy": "0.775", "provider_name": "Anthropic", "total_questions": "200"}, {"accuracy": "0.785", "provider_name": "Azure", "total_questions": "400"}, {"accuracy": "0.78", "provider_name": "Claude Platform on AWS", "total_questions": "200"}, {"accuracy": "0.7855555833333333", "provider_name": "Google", "total_questions": "600"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.56, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "0.56", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-10-13T13:41:42.000Z", "id": "N9uFn7EZFsvHNcHEwEy3J6", "mean_score": "0.56", "stderr": "0.049888765156985884"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-23", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5833333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "0.5833333333333334", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-23T20:34:23.000Z", "id": "aV7jRQtNYXjDAcUHhvKxAx", "mean_score": "0.5833333333333334", "stderr": "0.06418399868645064"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-22", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.56, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "0.56", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T19:59:24.000Z", "id": "DnFTaE9qNsmBdYNqohhynY", "mean_score": "0.56", "stderr": "0.049888765156985884"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-22", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "0.9", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T19:59:24.000Z", "id": "jwS3LYLCPbZpHDzzkX6LyV", "mean_score": "0.9", "stderr": "0.051739520574625435"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-22", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8982456140350877, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "0.8982456140350877", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T19:59:24.000Z", "id": "RHjHEPHEsWZqFuMjBTeC6r", "mean_score": "0.8982456140350877", "stderr": "0.01793969072128637"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-22", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T19:59:24.000Z", "id": "VZAUcNkzeMSMm9paxBhj6B", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-22", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9431818181818182, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "0.9431818181818182", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T19:59:24.000Z", "id": "aJD7ZfFftu2cZoToJhhzF7", "mean_score": "0.9431818181818182", "stderr": "0.014938512008159717"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9208749999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6-sol-20260922", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-sol-20260922", "rows": [{"accuracy": "0.9191915", "provider_name": "Amazon Bedrock", "total_questions": "396"}, {"accuracy": "0.924599700084246", "provider_name": "Azure", "total_questions": "1187"}, {"accuracy": "0.9208749999999999", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-22", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.607, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-sol_max", "source_row": {"Best score (across scorers)": "0.607", "Country": "United States of America", "Model version": "gpt-6-sol_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T19:59:24.000Z", "id": "donMeEKBZZdqpQsojKWJEm", "mean_score": "0.607", "stderr": "0.015452824654081525"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-6-sol", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7022223333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6-sol-20260922", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-sol-20260922", "rows": [{"accuracy": "0.6966665000000001", "provider_name": "Amazon Bedrock", "total_questions": "100"}, {"accuracy": "0.6706668", "provider_name": "Azure", "total_questions": "250"}, {"accuracy": "0.7022223333333333", "provider_name": "OpenAI", "total_questions": "150"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.59, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "0.59", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:15:18.000Z", "id": "GJtYeC7ZV4fYgN3wu4rRvq", "mean_score": "0.59", "stderr": "0.04943110704237103"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.725, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "0.725", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:15:28.000Z", "id": "bRw53PXBsZXrvWau95hb23", "mean_score": "0.725", "stderr": "0.056279417919483966"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.64, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "0.64", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:14:57.000Z", "id": "9ipFgNR5MDw7ustUsuSviu", "mean_score": "0.64", "stderr": "0.048241815132442176"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7804878048780488, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "0.7804878048780488", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:14:57.000Z", "id": "6FQCa8tMipf2ncdgGgMaPc", "mean_score": "0.7804878048780488", "stderr": "0.06544589202438408"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8596491228070176, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "0.8596491228070176", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:14:57.000Z", "id": "N8daAyKM6Sah39YBcPRi4W", "mean_score": "0.8596491228070176", "stderr": "0.020611471958164294"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:14:57.000Z", "id": "43f36RTDoJGxXWQDVvTLJj", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9640151515151515, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "0.9640151515151515", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:15:08.000Z", "id": "QEouTvczNsxgkR4i7i8wBF", "mean_score": "0.9640151515151515", "stderr": "0.011918129589602694"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-14", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.964646, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-4-argon-20261013", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-4-argon-20261013", "rows": [{"accuracy": "0.964646", "provider_name": "Google", "total_questions": "198"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-10-13", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.76, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-4-argon_high", "source_row": {"Best score (across scorers)": "0.76", "Country": "United States of America", "Model version": "gemini-4-argon_high", "Organization": "Google DeepMind", "Release date": "2026-09-30", "Started at": "2026-10-13T17:15:14.000Z", "id": "iq4Y9tXpiyLP25yotySxJW", "mean_score": "0.76", "stderr": "0.013512312258920845"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-14", "model": "gemini-4-argon", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.786667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-4-argon-20261013", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-4-argon-20261013", "rows": [{"accuracy": "0.786667", "provider_name": "Google", "total_questions": "50"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.59, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "0.59", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-10-13T13:50:12.000Z", "id": "oWqwMmwEd7LU3eU5Ftak64", "mean_score": "0.59", "stderr": "0.049431107042371025"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "0.5666666666666667", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-09-10T20:22:35.000Z", "id": "SqvYScyg9JZfiUdNLX6V9M", "mean_score": "0.5666666666666667", "stderr": "0.06340911315327727"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-28", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.58, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "0.58", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-28T20:58:29.000Z", "id": "5kMRXeasRvtghXpDotCdgV", "mean_score": "0.58", "stderr": "0.04960449637488583"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-07-09", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8292682926829268, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "0.8292682926829268", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T02:45:09.000Z", "id": "QJ5rJMVypB4PMxPcBGftc8", "mean_score": "0.8292682926829268", "stderr": "0.05949419959829497"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-07-09", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8912280701754386, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "0.8912280701754386", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T02:45:09.000Z", "id": "VBpT2WWYKBZtUxG9Z3svGh", "mean_score": "0.8912280701754386", "stderr": "0.018475392571140503"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-09", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T02:07:13.000Z", "id": "7RjsANKwwsePSstcZeSMhX", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-07-09", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9349747474747475, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "0.9349747474747475", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T02:07:13.000Z", "id": "nuKCgZReCBsXKCkNj9bK37", "mean_score": "0.9349747474747475", "stderr": "0.01570109070867593"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9234005000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.6-sol-20260709", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-sol-20260709", "rows": [{"accuracy": "0.9227054347826086", "provider_name": "Amazon Bedrock", "total_questions": "759"}, {"accuracy": "0.9248735", "provider_name": "Azure", "total_questions": "1584"}, {"accuracy": "0.9234005000000001", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.697, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-sol_max", "source_row": {"Best score (across scorers)": "0.697", "Country": "United States of America", "Model version": "gpt-5.6-sol_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-08-10T23:44:30.000Z", "id": "UnRUSuZzm2X27TUbpeMPva", "mean_score": "0.697", "stderr": "0.014539683710535193"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-6-sol", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.73333325, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.6-sol-20260709", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-sol-20260709", "rows": [{"accuracy": "0.7188551919191919", "provider_name": "Amazon Bedrock", "total_questions": "198"}, {"accuracy": "0.7403703333333334", "provider_name": "Azure", "total_questions": "450"}, {"accuracy": "0.73333325", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5-1_max", "source_row": {"Best score (across scorers)": "0.48", "Country": "United States of America", "Model version": "claude-fable-5-1_max", "Organization": "Anthropic", "Release date": "2026-09-01", "Started at": "2026-10-13T13:13:06.000Z", "id": "aLrHd8RCiNo3Yb4GqSawM7", "mean_score": "0.48", "stderr": "0.05021167315686782"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5-1_max", "source_row": {"Best score (across scorers)": "0.7", "Country": "United States of America", "Model version": "claude-fable-5-1_max", "Organization": "Anthropic", "Release date": "2026-09-01", "Started at": "2026-09-10T20:23:45.000Z", "id": "9LZMBmKxeBgnf5mvgNvBjW", "mean_score": "0.7", "stderr": "0.05724365605193373"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-01", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.58, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5-1_max", "source_row": {"Best score (across scorers)": "0.58", "Country": "United States of America", "Model version": "claude-fable-5-1_max", "Organization": "Anthropic", "Release date": "2026-09-01", "Started at": "2026-09-01T18:45:06.000Z", "id": "dBHxnnTzLGz3nHis5LET6t", "mean_score": "0.58", "stderr": "0.04960449637488583"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-01", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8780487804878049, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5-1_max", "source_row": {"Best score (across scorers)": "0.8780487804878049", "Country": "United States of America", "Model version": "claude-fable-5-1_max", "Organization": "Anthropic", "Release date": "2026-09-01", "Started at": "2026-09-01T18:38:24.000Z", "id": "T56N3X4KLA3xz7tf5aCMGb", "mean_score": "0.8780487804878049", "stderr": "0.051739520574625435"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-01", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9017543859649123, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5-1_max", "source_row": {"Best score (across scorers)": "0.9017543859649123", "Country": "United States of America", "Model version": "claude-fable-5-1_max", "Organization": "Anthropic", "Release date": "2026-09-01", "Started at": "2026-09-01T18:38:24.000Z", "id": "g7dXwZH2tWvqLg3X5idGs2", "mean_score": "0.9017543859649123", "stderr": "0.017662067789031757"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-01", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5-1_max", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "claude-fable-5-1_max", "Organization": "Anthropic", "Release date": "2026-09-01", "Started at": "2026-09-01T18:32:00.000Z", "id": "mRAASXsUuoJmc2oR5G89zR", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8894498333333334, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-fable-5.1-20260831", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "rows": [{"accuracy": "0.8894498333333334", "provider_name": "Anthropic", "total_questions": "1188"}, {"accuracy": "0.8872051666666666", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.8765431666666668", "provider_name": "Google", "total_questions": "1188"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-01", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.708, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5-1_max", "source_row": {"Best score (across scorers)": "0.708", "Country": "United States of America", "Model version": "claude-fable-5-1_max", "Organization": "Anthropic", "Release date": "2026-09-01", "Started at": "2026-09-01T18:31:59.000Z", "id": "ExyL3GN8s34tizrXV3VmS3", "mean_score": "0.708", "stderr": "0.014385511563477433"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "claude-fable-5-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7906666, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-fable-5.1-20260831", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-fable-5.1-20260831", "rows": [{"accuracy": "0.7906666", "provider_name": "Anthropic", "total_questions": "250"}, {"accuracy": "0.772", "provider_name": "Azure", "total_questions": "250"}, {"accuracy": "0.7746668", "provider_name": "Google", "total_questions": "250"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.63, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5_xhigh", "source_row": {"Best score (across scorers)": "0.63", "Country": "United States of America", "Model version": "gpt-5.5_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-10-13T13:54:48.000Z", "id": "DFWQcrbMegFAcEWZKWoJxC", "mean_score": "0.63", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.44166666666666665, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5_xhigh", "source_row": {"Best score (across scorers)": "0.44166666666666665", "Country": "United States of America", "Model version": "gpt-5.5_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-09-10T20:22:27.000Z", "id": "o7rYVQWSXeZPkF2Smwo9h8", "mean_score": "0.44166666666666665", "stderr": "0.06410142328406845"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-24", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.56, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5_xhigh", "source_row": {"Best score (across scorers)": "0.56", "Country": "United States of America", "Model version": "gpt-5.5_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-07-24T18:19:51.000Z", "id": "EatRQnSaUpUwKzdtjYnQHo", "mean_score": "0.56", "stderr": "0.049888765156985884"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.725, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5_xhigh", "source_row": {"Best score (across scorers)": "0.725", "Country": "United States of America", "Model version": "gpt-5.5_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-06-11T17:47:36.000Z", "id": "PKSMspJiJdNThB6qNvaBSM", "mean_score": "0.725", "stderr": "0.07149950690165274"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8526315789473684, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5_xhigh", "source_row": {"Best score (across scorers)": "0.8526315789473684", "Country": "United States of America", "Model version": "gpt-5.5_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-06-11T17:47:36.000Z", "id": "5FGXKvXQwGPtcCX2PNrLR5", "mean_score": "0.8526315789473684", "stderr": "0.021034091168852485"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-04-24", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pre-release_xhigh", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gpt-5.5-pre-release_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-04-24T14:06:46.000Z", "id": "dP6L7ViZSdcTiWxooLCTUh", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-04-24", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8057851239669421, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pre-release_xhigh", "source_row": {"Best score (across scorers)": "0.8057851239669421", "Country": "United States of America", "Model version": "gpt-5.5-pre-release_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-04-24T13:52:36.000Z", "id": "LbhPRqUkXrs5KJ7vyT4CR7", "mean_score": "0.8057851239669421", "stderr": "0.018000198869416815"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-04-24", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9400252525252525, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pre-release_xhigh", "source_row": {"Best score (across scorers)": "0.9400252525252525", "Country": "United States of America", "Model version": "gpt-5.5-pre-release_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-04-24T13:35:12.000Z", "id": "FUv4EUwR78nAsdGbiwiLEr", "mean_score": "0.9400252525252525", "stderr": "0.015469771177277509"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9360266666666668, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.5-20260423", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.5-20260423", "rows": [{"accuracy": "0.9255440309278352", "provider_name": "Amazon Bedrock", "total_questions": "582"}, {"accuracy": "0.9326596250000001", "provider_name": "Azure", "total_questions": "1584"}, {"accuracy": "0.9360266666666668", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.63, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5_xhigh", "source_row": {"Best score (across scorers)": "0.63", "Country": "United States of America", "Model version": "gpt-5.5_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-08-27T19:28:25.000Z", "id": "TijCafTTJUFrJ6jPGvBbgq", "mean_score": "0.63", "stderr": "0.015275252316519465"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.75, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.5-20260423", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.5-20260423", "rows": [{"accuracy": "0.7466665", "provider_name": "Amazon Bedrock", "total_questions": "100"}, {"accuracy": "0.7525925555555556", "provider_name": "Azure", "total_questions": "450"}, {"accuracy": "0.75", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.47, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_max", "source_row": {"Best score (across scorers)": "0.47", "Country": "United States of America", "Model version": "claude-fable-5_max", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-10-13T13:34:37.000Z", "id": "bxS76CJuVws8iwWU2Wr2PM", "mean_score": "0.47", "stderr": "0.05016135580465918"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.35833333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_max", "source_row": {"Best score (across scorers)": "0.35833333333333334", "Country": "United States of America", "Model version": "claude-fable-5_max", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-09-10T20:23:38.000Z", "id": "433Hqw9fVgtRtAKQfareFr", "mean_score": "0.35833333333333334", "stderr": "0.06185876325418331"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-31", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.52, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_max", "source_row": {"Best score (across scorers)": "0.52", "Country": "United States of America", "Model version": "claude-fable-5_max", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-07-31T19:05:04.000Z", "id": "X29V6KG3aVe5Nv5r3tsLHX", "mean_score": "0.52", "stderr": "0.05021167315686782"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-09", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.902, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_max", "source_row": {"Best score (across scorers)": "0.902", "Country": "United States of America", "Model version": "claude-fable-5_max", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-06-09T18:12:35.000Z", "id": "ReDAtJGGfRtrEGGWf4Zbsd", "mean_score": "0.902", "stderr": "0.046"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-09", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8701754385964913, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_max", "source_row": {"Best score (across scorers)": "0.8701754385964913", "Country": "United States of America", "Model version": "claude-fable-5_max", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-06-09T18:12:35.000Z", "id": "E28uZiJvrPgKtawbJpoZQ8", "mean_score": "0.8701754385964913", "stderr": "0.019944477920133024"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-06", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_high", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "claude-fable-5_high", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-08-06T22:18:19.000Z", "id": "hwrnvtEbDAocFtjcsHqsQh", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8585858585858586, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_max", "source_row": {"Best score (across scorers)": "0.8585858585858586", "Country": "United States of America", "Model version": "claude-fable-5_max", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-08-06T22:17:56.000Z", "id": "S5pm7ctk9zgPfbyHrbD9AV", "mean_score": "0.8585858585858586", "stderr": "0.024825909793343353"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8615317499999998, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-5-fable-20260609", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-5-fable-20260609", "rows": [{"accuracy": "0.8181816666666667", "provider_name": "Amazon Bedrock", "total_questions": "594"}, {"accuracy": "0.8615317499999998", "provider_name": "Anthropic", "total_questions": "2376"}, {"accuracy": "0.8574634166666667", "provider_name": "Azure", "total_questions": "2376"}, {"accuracy": "0.8663297", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.8533156956521739", "provider_name": "Google", "total_questions": "4554"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.707, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-fable-5_xhigh", "source_row": {"Best score (across scorers)": "0.707", "Country": "United States of America", "Model version": "claude-fable-5_xhigh", "Organization": "Anthropic", "Release date": "2026-06-09", "Started at": "2026-08-10T23:44:03.000Z", "id": "SEnvgtunRa5GwAtU2sHa87", "mean_score": "0.707", "stderr": "0.014399942998441292"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-fable-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7957576363636364, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-5-fable-20260609", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-5-fable-20260609", "rows": [{"accuracy": "0.8", "provider_name": "Amazon Bedrock", "total_questions": "150"}, {"accuracy": "0.7957576363636364", "provider_name": "Anthropic", "total_questions": "550"}, {"accuracy": "0.8054545454545454", "provider_name": "Azure", "total_questions": "550"}, {"accuracy": "0.8022223333333333", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7944445714285714", "provider_name": "Google", "total_questions": "1050"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.43, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.43", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-10-13T13:19:00.000Z", "id": "RLN98bmQSMnc5Zv39BF4M2", "mean_score": "0.43", "stderr": "0.0497569851956243"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6083333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.6083333333333333", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-09-10T20:23:29.000Z", "id": "eBTo3dRKwCVuHaRSyN24VA", "mean_score": "0.6083333333333333", "stderr": "0.06185876325418331"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-25", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.59, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.59", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-07-25T04:37:04.000Z", "id": "FfAvNS5H9bMDSBF4JNNiDz", "mean_score": "0.59", "stderr": "0.04943110704237103"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-07-24", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7317073170731707, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.7317073170731707", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-07-24T19:54:20.000Z", "id": "ji3kMn49Jcr9t4whxcpjj4", "mean_score": "0.7317073170731707", "stderr": "0.07005564203095158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-07-24", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.856140350877193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.856140350877193", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-07-24T19:54:20.000Z", "id": "ff7GeS7rdperjxSBZ7MHZY", "mean_score": "0.856140350877193", "stderr": "0.020824894575343943"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-24", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9888888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.9888888888888889", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-07-24T18:40:39.000Z", "id": "KxvHAT9SV2dTDn7jBnTk6f", "mean_score": "0.9888888888888889", "stderr": "0.011111111111111112"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-07-24", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9387626262626263, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.9387626262626263", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-07-24T18:40:39.000Z", "id": "D3mtasTumfvjR4fjmnHWRy", "mean_score": "0.9387626262626263", "stderr": "0.014782015056089346"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8957629166666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-opus-5-20260723", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-opus-5-20260723", "rows": [{"accuracy": "0.8819856633763972", "provider_name": "Amazon Bedrock", "total_questions": "6889"}, {"accuracy": "0.8957629166666667", "provider_name": "Anthropic", "total_questions": "2376"}, {"accuracy": "0.8972450554493306", "provider_name": "Azure", "total_questions": "4707"}, {"accuracy": "0.8804433325415677", "provider_name": "Claude Platform on AWS", "total_questions": "2526"}, {"accuracy": "0.8900633006342494", "provider_name": "Google", "total_questions": "7095"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.599, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-5_max", "source_row": {"Best score (across scorers)": "0.599", "Country": "United States of America", "Model version": "claude-opus-5_max", "Organization": "Anthropic", "Release date": "2026-07-24", "Started at": "2026-08-10T23:44:04.000Z", "id": "Zftz2yk4X5NF28JQzjN3hZ", "mean_score": "0.599", "stderr": "0.01550610974549837"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-opus-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7727272727272727, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-opus-5-20260723", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-opus-5-20260723", "rows": [{"accuracy": "0.778383818181818", "provider_name": "Amazon Bedrock", "total_questions": "1650"}, {"accuracy": "0.7727272727272727", "provider_name": "Anthropic", "total_questions": "550"}, {"accuracy": "0.7818182727272727", "provider_name": "Azure", "total_questions": "1100"}, {"accuracy": "0.77666675", "provider_name": "Claude Platform on AWS", "total_questions": "600"}, {"accuracy": "0.7801960294117648", "provider_name": "Google", "total_questions": "1700"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.47, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.47", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-10-13T13:53:13.000Z", "id": "J2ZgtxyaQZfyDMPTPxS8nm", "mean_score": "0.47", "stderr": "0.05016135580465918"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5416666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.5416666666666666", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-09-10T20:22:34.000Z", "id": "dxcm8tfHzGWR8sVJ6kUWMF", "mean_score": "0.5416666666666666", "stderr": "0.06432138878020231"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-28", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.35, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.35", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-28T20:58:46.000Z", "id": "MFTd4BUbtzbfcCky6d79d4", "mean_score": "0.35", "stderr": "0.047937248544110224"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-07-09", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7073170731707317, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.7073170731707317", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T15:58:30.000Z", "id": "ZUygpkXqg3NjchXXfedAWV", "mean_score": "0.7073170731707317", "stderr": "0.07194088392074452"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-07-09", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8596491228070176, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.8596491228070176", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T15:58:30.000Z", "id": "J6evjTMgHkrun8txnEvzdN", "mean_score": "0.8596491228070176", "stderr": "0.020611471958164294"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-09", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9972222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.9972222222222222", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T17:07:54.000Z", "id": "jUmmCNezoW7jWiK72SVCaP", "mean_score": "0.9972222222222222", "stderr": "0.002777777777777778"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-07-09", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9330808080808081, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.9330808080808081", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T17:07:54.000Z", "id": "4bLcFNu5zF6YuZf3qVKoHE", "mean_score": "0.9330808080808081", "stderr": "0.015363672147634115"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.89267675, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.6-terra-20260709", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-terra-20260709", "rows": [{"accuracy": "0.8950130708661417", "provider_name": "Amazon Bedrock", "total_questions": "762"}, {"accuracy": "0.8941264444444444", "provider_name": "Azure", "total_questions": "1782"}, {"accuracy": "0.89267675", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.432, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-terra_max", "source_row": {"Best score (across scorers)": "0.432", "Country": "United States of America", "Model version": "gpt-5.6-terra_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-08-10T23:44:21.000Z", "id": "DienatPbgfgn3EzM7Gvzxr", "mean_score": "0.432", "stderr": "0.015672320237336324"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-6-terra", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.715, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.6-terra-20260709", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-terra-20260709", "rows": [{"accuracy": "0.71", "provider_name": "Amazon Bedrock", "total_questions": "200"}, {"accuracy": "0.7170371111111111", "provider_name": "Azure", "total_questions": "450"}, {"accuracy": "0.715", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.52, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.52", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-10-13T13:35:27.000Z", "id": "PEZpuEuU65bu3ZtvjLytHq", "mean_score": "0.52", "stderr": "0.05021167315686782"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.425, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.425", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-09-10T20:23:25.000Z", "id": "mTRwnGKK2QWHrovdPrrMbJ", "mean_score": "0.425", "stderr": "0.06380695638890652"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-25", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.36, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.36", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-07-25T04:37:20.000Z", "id": "Q3WpHdurYNvawj6ycXyj6e", "mean_score": "0.36", "stderr": "0.04824181513244218"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-10", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5609756097560976, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.5609756097560976", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-06-10T15:25:15.000Z", "id": "JiqgqBTfkGx797uowZgJbk", "mean_score": "0.5609756097560976", "stderr": "0.07846686801046543"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-10", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.8", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-06-10T15:25:15.000Z", "id": "GVmgtgKMwEqc54f3jA7auP", "mean_score": "0.8", "stderr": "0.023735633163877067"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-06-07", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9833333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.9833333333333333", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-06-07T23:44:54.000Z", "id": "37pJqPpXTE6ia9ikmmGPJ4", "mean_score": "0.9833333333333333", "stderr": "0.014101901870444154"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-06-07", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9103535353535354, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.9103535353535354", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-06-07T23:03:11.000Z", "id": "dqgqP2uEPswfECCVj2Kbr9", "mean_score": "0.9103535353535354", "stderr": "0.019157125794612026"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8959834999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.8-opus-20260528", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "rows": [{"accuracy": "0.9008713529411765", "provider_name": "Amazon Bedrock", "total_questions": "6732"}, {"accuracy": "0.8959834999999999", "provider_name": "Anthropic", "total_questions": "2772"}, {"accuracy": "0.901208909090909", "provider_name": "Azure", "total_questions": "4356"}, {"accuracy": "0.8974747000000001", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.8995188857142861", "provider_name": "Google", "total_questions": "6930"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.53, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-8_max", "source_row": {"Best score (across scorers)": "0.53", "Country": "United States of America", "Model version": "claude-opus-4-8_max", "Organization": "Anthropic", "Release date": "2026-05-28", "Started at": "2026-08-27T19:27:24.000Z", "id": "ViC6pxQZUVS3E9M9iEubF7", "mean_score": "0.53", "stderr": "0.015790799515836722"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-opus-4-8", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7776190714285715, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.8-opus-20260528", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.8-opus-20260528", "rows": [{"accuracy": "0.7681249375000001", "provider_name": "Amazon Bedrock", "total_questions": "1600"}, {"accuracy": "0.7776190714285715", "provider_name": "Anthropic", "total_questions": "700"}, {"accuracy": "0.776825380952381", "provider_name": "Azure", "total_questions": "1050"}, {"accuracy": "0.78", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7700000312500004", "provider_name": "Google", "total_questions": "1600"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.66, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.66", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-10-13T13:00:23.000Z", "id": "Xu34dKTAdpBWL6advf6DMg", "mean_score": "0.66", "stderr": "0.04760952285695234"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26666666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.26666666666666666", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-09-10T20:23:57.000Z", "id": "bqeNkcnEXJeZ9jgmao393V", "mean_score": "0.26666666666666666", "stderr": "0.056331678510110804"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-14", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.37", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-08-14T21:35:50.000Z", "id": "ZavSMMq9jk5s9WSbARZr6X", "mean_score": "0.37", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-14", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.36585365853658536, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.36585365853658536", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-08-14T19:59:58.000Z", "id": "KptKXK5nE2Ns2WbiaSSnbc", "mean_score": "0.36585365853658536", "stderr": "0.07615851217559022"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-14", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7157894736842105, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.7157894736842105", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-08-14T19:59:58.000Z", "id": "axMdgNDC8UYYLnZYLHZpF5", "mean_score": "0.7157894736842105", "stderr": "0.026764156649364653"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-14", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9722222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.9722222222222222", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-08-14T18:29:12.000Z", "id": "kQjrYCaa7yYyMXofeR23Dd", "mean_score": "0.9722222222222222", "stderr": "0.018586784946171872"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-14", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9482323232323232, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.9482323232323232", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-08-14T18:29:12.000Z", "id": "XyVL3VffJ6UboY97vxpohL", "mean_score": "0.9482323232323232", "stderr": "0.013456525292246259"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-15", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.9391951653543308, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.7-flash-20260813", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.7-flash-20260813", "rows": [{"accuracy": "0.9391951653543308", "provider_name": "Google", "total_questions": "762"}, {"accuracy": "0.9427606666666666", "provider_name": "Google AI Studio", "total_questions": "594"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.692, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.7-flash_high", "source_row": {"Best score (across scorers)": "0.692", "Country": "United States of America", "Model version": "gemini-3.7-flash_high", "Organization": "Google DeepMind", "Release date": "2026-08-13", "Started at": "2026-08-27T19:30:25.000Z", "id": "84ARuGyhQsjgh8umRmBvas", "mean_score": "0.692", "stderr": "0.01460648312734278"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-15", "model": "gemini-3-7-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7831052054794521, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.7-flash-20260813", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.7-flash-20260813", "rows": [{"accuracy": "0.7831052054794521", "provider_name": "Google", "total_questions": "146"}, {"accuracy": "0.806667", "provider_name": "Google AI Studio", "total_questions": "50"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.48", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-10-13T14:04:04.000Z", "id": "92ZBraB6bB98s8raSiPgUw", "mean_score": "0.48", "stderr": "0.05021167315686782"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.375, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.375", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-09-10T20:22:11.000Z", "id": "bJ5Pg8JiLzP3Wfoo9M9TxF", "mean_score": "0.375", "stderr": "0.06246467928511352"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-24", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.37", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-07-24T18:20:01.000Z", "id": "T6DyawEuhVmDAaDTzDRNNx", "mean_score": "0.37", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.49, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.49", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-06-11T17:55:45.000Z", "id": "doiQDghqnLzbE8CFijLNQJ", "mean_score": "0.49", "stderr": "0.08"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7859649122807018, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.7859649122807018", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-06-11T17:55:45.000Z", "id": "PaYEQ37A2n6giYBwZGBnXY", "mean_score": "0.7859649122807018", "stderr": "0.024338000553262393"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-15", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9777777777777777, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_high", "source_row": {"Best score (across scorers)": "0.9777777777777777", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_high", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-07-15T03:14:50.000Z", "id": "DpPahiEawokzgidE3cNZbS", "mean_score": "0.9777777777777777", "stderr": "0.022222222222222223"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-03-06", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.768595041322314, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_high", "source_row": {"Best score (across scorers)": "0.768595041322314", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_high", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-03-06T11:07:43.396Z", "id": "i9HTMbAxcwsrTf2w2FuJ6N", "mean_score": "0.768595041322314", "stderr": "0.019189407003691976"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-03-06", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.933, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.933", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-03-06T02:57:02.864Z", "id": "fFatyce8UvpN7ZivdmrhAy", "mean_score": "0.933", "stderr": "0.018"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.899551, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.4-20260305", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-20260305", "rows": [{"accuracy": "0.8968568964143425", "provider_name": "Amazon Bedrock", "total_questions": "753"}, {"accuracy": "0.8999248888888889", "provider_name": "Azure", "total_questions": "1782"}, {"accuracy": "0.899551", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.451, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.451", "Country": "United States of America", "Model version": "gpt-5.4-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-08-27T19:28:16.000Z", "id": "AzCkm282PLhDdLJ8DnAQQc", "mean_score": "0.451", "stderr": "0.015743152379585443"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-4", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.73166675, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.4-20260305", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-20260305", "rows": [{"accuracy": "0.7266665", "provider_name": "Amazon Bedrock", "total_questions": "200"}, {"accuracy": "0.7385184444444445", "provider_name": "Azure", "total_questions": "450"}, {"accuracy": "0.73166675", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.43, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.43", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-10-13T13:53:45.000Z", "id": "LUfGWqWQim7rdBPgSpbrXS", "mean_score": "0.43", "stderr": "0.0497569851956243"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.425, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.425", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-09-10T20:22:42.000Z", "id": "UPAoeCG9AhQYyxCGc8C7Eq", "mean_score": "0.425", "stderr": "0.06380695638890652"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-28", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.21", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-28T20:59:07.000Z", "id": "Epg7eNLwwGPXtX3sSAwPf8", "mean_score": "0.21", "stderr": "0.040936018074033236"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-07-09", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6097560975609756, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.6097560975609756", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T14:18:35.000Z", "id": "NLiiJJSyA9wmg2FHENFpAq", "mean_score": "0.6097560975609756", "stderr": "0.07712872341874097"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-07-09", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8210526315789474, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.8210526315789474", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T14:18:35.000Z", "id": "YLAk5J3Wd8EruVXee8bAQb", "mean_score": "0.8210526315789474", "stderr": "0.02274515950332741"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-09", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9833333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.9833333333333333", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T13:36:49.000Z", "id": "AhtQ5ggs4pTfLDqHFjueGy", "mean_score": "0.9833333333333333", "stderr": "0.011650421590325524"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-07-09", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9160353535353535, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.9160353535353535", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-07-09T13:36:49.000Z", "id": "LWkDtkmd3TNmLzMhfZD5HW", "mean_score": "0.9160353535353535", "stderr": "0.01728284355494035"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8724745, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.6-luna-20260709", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.6-luna-20260709", "rows": [{"accuracy": "0.8853190481927712", "provider_name": "Amazon Bedrock", "total_questions": "747"}, {"accuracy": "0.884269595505618", "provider_name": "Azure", "total_questions": "1780"}, {"accuracy": "0.8724745", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.41, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.6-luna_max", "source_row": {"Best score (across scorers)": "0.41", "Country": "United States of America", "Model version": "gpt-5.6-luna_max", "Organization": "OpenAI", "Release date": "2026-07-09", "Started at": "2026-08-10T23:44:20.000Z", "id": "CeDghdBk8CqtiFSbUzxNH5", "mean_score": "0.41", "stderr": "0.015560917136921657"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-6-luna", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.72333325, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.6-luna-20260709", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.6-luna-20260709", "rows": [{"accuracy": "0.6816665000000001", "provider_name": "Amazon Bedrock", "total_questions": "200"}, {"accuracy": "0.7103705555555555", "provider_name": "Azure", "total_questions": "450"}, {"accuracy": "0.72333325", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.36, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.36", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-10-13T13:37:06.000Z", "id": "Jevnk9hi9oRBbGcFu2JxBC", "mean_score": "0.36", "stderr": "0.04824181513244218"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-28", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.44166666666666665, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.44166666666666665", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-28T03:08:57.000Z", "id": "GVtcaKADdbkuPoydqpKm3E", "mean_score": "0.44166666666666665", "stderr": "0.06410142328406845"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-22", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.07, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.07", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T22:14:05.000Z", "id": "bAERV8wdmoHMLz2K4bZCSK", "mean_score": "0.07", "stderr": "0.02564323999762428"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-22", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5609756097560976, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.5609756097560976", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T22:14:05.000Z", "id": "7n3HBUK684GmdT6Tkycu6m", "mean_score": "0.5609756097560976", "stderr": "0.07846686801046543"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-22", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7894736842105263, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.7894736842105263", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T22:14:05.000Z", "id": "iBQxysYR7zPxB6hxvJyTLc", "mean_score": "0.7894736842105263", "stderr": "0.024191503146153773"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-22", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9888888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.9888888888888889", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T22:14:05.000Z", "id": "4uDz9b6SosxDSZf7swMD9D", "mean_score": "0.9888888888888889", "stderr": "0.006674027922425511"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-22", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9046717171717171, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.9046717171717171", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T22:14:05.000Z", "id": "7F3QFMAkwnCV62HcvxP96q", "mean_score": "0.9046717171717171", "stderr": "0.018537378631915503"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8815936666666665, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6-luna-20260922", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-6-luna-20260922", "rows": [{"accuracy": "0.880471", "provider_name": "Amazon Bedrock", "total_questions": "198"}, {"accuracy": "0.8771043333333335", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.8815936666666665", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-22", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.414, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-6-luna_max", "source_row": {"Best score (across scorers)": "0.414", "Country": "United States of America", "Model version": "gpt-6-luna_max", "Organization": "OpenAI", "Release date": "2026-09-22", "Started at": "2026-09-22T22:14:05.000Z", "id": "HX9sRVWJtZiNvneALnVzs8", "mean_score": "0.414", "stderr": "0.015583544104177553"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-6-luna", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.68, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-6-luna-20260922", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-6-luna-20260922", "rows": [{"accuracy": "0.653333", "provider_name": "Amazon Bedrock", "total_questions": "50"}, {"accuracy": "0.6811111666666666", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.68", "provider_name": "OpenAI", "total_questions": "150"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.58, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.58", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-10-13T12:39:05.000Z", "id": "Hpyy6CpmXmd5q7Zbdq63CR", "mean_score": "0.58", "stderr": "0.04960449637488583"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.31666666666666665, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.31666666666666665", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-09-10T20:23:47.000Z", "id": "Q2QhUc6V3Cu9XS3WZH4tQh", "mean_score": "0.31666666666666665", "stderr": "0.0593832076297605"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-02", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.47, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.47", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-09-02T17:15:36.000Z", "id": "5yyuCCi2Dpfh6xLBMxqFyB", "mean_score": "0.47", "stderr": "0.05016135580465918"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-02", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21951219512195122, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.21951219512195122", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-09-02T17:15:36.000Z", "id": "Lgy52ZT4YpFrUqqKVu2qUs", "mean_score": "0.21951219512195122", "stderr": "0.06544589202438408"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-02", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6842105263157895, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.6842105263157895", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-09-02T17:15:36.000Z", "id": "Utgs6DuaQDDdi98ZWH22qL", "mean_score": "0.6842105263157895", "stderr": "0.027582557383453006"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-02", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9888888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.9888888888888889", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-09-02T17:15:36.000Z", "id": "NeYuqgQyWtFWAHEjQYPm5Z", "mean_score": "0.9888888888888889", "stderr": "0.008724007409153888"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-02", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9539141414141414, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.9539141414141414", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-09-02T17:15:36.000Z", "id": "Br3mjs2GrYcCwX9AMwcihU", "mean_score": "0.9539141414141414", "stderr": "0.01401136318210652"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-15", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.9482614885496183, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.8-flash-20260902", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.8-flash-20260902", "rows": [{"accuracy": "0.9482614885496183", "provider_name": "Google", "total_questions": "393"}, {"accuracy": "0.952862", "provider_name": "Google AI Studio", "total_questions": "198"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-02", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.697, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.8-flash_high", "source_row": {"Best score (across scorers)": "0.697", "Country": "United States of America", "Model version": "gemini-3.8-flash_high", "Organization": "Google DeepMind", "Release date": "2026-09-02", "Started at": "2026-09-02T17:15:36.000Z", "id": "gZ4bwozTKtpmZTqnqX3a33", "mean_score": "0.697", "stderr": "0.014539683710535193"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-3-8-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.734694, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.8-flash-20260902", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.8-flash-20260902", "rows": [{"accuracy": "0.7666665", "provider_name": "Google", "total_questions": "100"}, {"accuracy": "0.734694", "provider_name": "Google AI Studio", "total_questions": "49"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.55, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_max", "source_row": {"Best score (across scorers)": "0.55", "Country": "United States of America", "Model version": "claude-haiku-5-5_max", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-13T12:40:01.000Z", "id": "KEyuvehfRGtcdgNfNavy62", "mean_score": "0.55", "stderr": "0.049999999999999996"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-10-07", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.475, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_max", "source_row": {"Best score (across scorers)": "0.475", "Country": "United States of America", "Model version": "claude-haiku-5-5_max", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-07T20:22:18.000Z", "id": "bZMMyetG9pAbKgpxghoTe8", "mean_score": "0.475", "stderr": "0.06446761548504554"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-10-07", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_max", "source_row": {"Best score (across scorers)": "0.3", "Country": "United States of America", "Model version": "claude-haiku-5-5_max", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-07T20:51:20.000Z", "id": "CKm7NWXfSePfcUycRK87cB", "mean_score": "0.3", "stderr": "0.04605661864718382"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-10-07", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4634146341463415, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_max", "source_row": {"Best score (across scorers)": "0.4634146341463415", "Country": "United States of America", "Model version": "claude-haiku-5-5_max", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-07T20:42:31.000Z", "id": "WmUXFQoq7GqiSgpTmfLaVv", "mean_score": "0.4634146341463415", "stderr": "0.07884502378168105"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-10-07", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7508771929824561, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_max", "source_row": {"Best score (across scorers)": "0.7508771929824561", "Country": "United States of America", "Model version": "claude-haiku-5-5_max", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-07T20:42:31.000Z", "id": "2QAt4ptgGkLtcbVL3hdAvL", "mean_score": "0.7508771929824561", "stderr": "0.02566445410232946"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-10-07", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9888888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_xhigh", "source_row": {"Best score (across scorers)": "0.9888888888888889", "Country": "United States of America", "Model version": "claude-haiku-5-5_xhigh", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-07T21:16:36.000Z", "id": "iiEPrSjWXHH6Zq9CK4Zmcr", "mean_score": "0.9888888888888889", "stderr": "0.005362818328685681"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-10-09", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8964646464646465, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_max", "source_row": {"Best score (across scorers)": "0.8964646464646465", "Country": "United States of America", "Model version": "claude-haiku-5-5_max", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-09T14:35:53.000Z", "id": "ZEKWTGJiWFscHimMkiUmec", "mean_score": "0.8964646464646465", "stderr": "0.019774484382483127"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8692479999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-haiku-5.5-20261007", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "rows": [{"accuracy": "0.8720537777777777", "provider_name": "Amazon Bedrock", "total_questions": "1782"}, {"accuracy": "0.8692479999999999", "provider_name": "Anthropic", "total_questions": "594"}, {"accuracy": "0.8672838333333334", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.8720540000000001", "provider_name": "Claude Platform on AWS", "total_questions": "594"}, {"accuracy": "0.8677515555555556", "provider_name": "Google", "total_questions": "1782"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-10-09", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.238, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-5-5_max", "source_row": {"Best score (across scorers)": "0.238", "Country": "United States of America", "Model version": "claude-haiku-5-5_max", "Organization": "Anthropic", "Release date": "2026-10-07", "Started at": "2026-10-09T14:36:06.000Z", "id": "NwnrUMtveqHDYB2Um4ycmA", "mean_score": "0.238", "stderr": "0.013473586661967093"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-haiku-5-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7577776666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-haiku-5.5-20261007", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-haiku-5.5-20261007", "rows": [{"accuracy": "0.7474073333333333", "provider_name": "Amazon Bedrock", "total_questions": "450"}, {"accuracy": "0.7577776666666667", "provider_name": "Anthropic", "total_questions": "150"}, {"accuracy": "0.7444443333333334", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.748889", "provider_name": "Claude Platform on AWS", "total_questions": "150"}, {"accuracy": "0.7466665555555556", "provider_name": "Google", "total_questions": "450"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.48", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-13T12:39:44.000Z", "Training compute (FLOP)": "4.32e+24", "id": "ACpDTbhRPwce2BWQTWD8iC", "mean_score": "0.48", "stderr": "0.05021167315686782"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-10-08", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3416666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.3416666666666667", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-08T19:02:18.000Z", "Training compute (FLOP)": "4.32e+24", "id": "JFQz7q3hxAKLV5pW6DmkGF", "mean_score": "0.3416666666666667", "stderr": "0.06000431560123094"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-10-08", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.43, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.43", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-08T19:02:32.000Z", "Training compute (FLOP)": "4.32e+24", "id": "K3JABAsJCbXU5YrCPH2yVW", "mean_score": "0.43", "stderr": "0.0497569851956243"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-10-08", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2682926829268293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.2682926829268293", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-08T19:02:32.000Z", "Training compute (FLOP)": "4.32e+24", "id": "KansJcyeGR4m7G4p8GupDn", "mean_score": "0.2682926829268293", "stderr": "0.07005564203095158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-10-08", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6736842105263158, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.6736842105263158", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-08T19:02:32.000Z", "Training compute (FLOP)": "4.32e+24", "id": "b6dXtaGSwvfRYADKYnj9Ci", "mean_score": "0.6736842105263158", "stderr": "0.027821981570324446"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-10-08", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9833333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.9833333333333333", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-08T19:02:32.000Z", "Training compute (FLOP)": "4.32e+24", "id": "7eHNiVQCzLcAyj5LxmeeSU", "mean_score": "0.9833333333333333", "stderr": "0.006405884289881728"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-10-09", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8983585858585859, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.8983585858585859", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-09T14:38:01.000Z", "Training compute (FLOP)": "4.32e+24", "id": "ZU5ebpDaDPiDZLgv7cG8B9", "mean_score": "0.8983585858585859", "stderr": "0.017287759372442393"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.8838381666666666, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4.1-flash-20260910", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "rows": [{"accuracy": "0.904882", "provider_name": "Alibaba", "total_questions": "1188"}, {"accuracy": "0.8900111666666666", "provider_name": "AtlasCloud", "total_questions": "1188"}, {"accuracy": "0.9048820000000001", "provider_name": "Baidu", "total_questions": "792"}, {"accuracy": "0.8640821969539633", "provider_name": "BaseTen", "total_questions": "2889"}, {"accuracy": "0.8754205", "provider_name": "Cloudflare", "total_questions": "396"}, {"accuracy": "0.8634711176470588", "provider_name": "CoreWeave", "total_questions": "918"}, {"accuracy": "0.8681256666666667", "provider_name": "Crusoe", "total_questions": "594"}, {"accuracy": "0.8843901497975709", "provider_name": "Decart", "total_questions": "741"}, {"accuracy": "0.9030148159392788", "provider_name": "DeepInfra", "total_questions": "1581"}, {"accuracy": "0.8850689656218401", "provider_name": "DekaLLM", "total_questions": "989"}, {"accuracy": "0.8973243579175705", "provider_name": "DigitalOcean", "total_questions": "1383"}, {"accuracy": "0.8731974290076335", "provider_name": "Fireworks", "total_questions": "1965"}, {"accuracy": "0.8917747142857142", "provider_name": "GMICloud", "total_questions": "1386"}, {"accuracy": "0.8813526750788644", "provider_name": "InferenceNet", "total_questions": "951"}, {"accuracy": "0.8397870000000001", "provider_name": "Io Net", "total_questions": "594"}, {"accuracy": "0.88173375", "provider_name": "Ionstream", "total_questions": "792"}, {"accuracy": "0.7234044255319149", "provider_name": "Krea", "total_questions": "517"}, {"accuracy": "0.8732769164556962", "provider_name": "Makora", "total_questions": "1185"}, {"accuracy": "0.8790683333333332", "provider_name": "Modal", "total_questions": "1188"}, {"accuracy": "0.8886081666666666", "provider_name": "Morph", "total_questions": "1188"}, {"accuracy": "0.8838380000000001", "provider_name": "Nebius", "total_questions": "396"}, {"accuracy": "0.9094272", "provider_name": "NextBit", "total_questions": "990"}, {"accuracy": "0.9028881689530687", "provider_name": "Novita", "total_questions": "1385"}, {"accuracy": "0.8646462", "provider_name": "OpenInference", "total_questions": "990"}, {"accuracy": "0.8813129999999999", "provider_name": "Parasail", "total_questions": "1188"}, {"accuracy": "0.8910815631578948", "provider_name": "Phala", "total_questions": "1140"}, {"accuracy": "0.9017853214285716", "provider_name": "PrimeIntellect", "total_questions": "392"}, {"accuracy": "0.879189", "provider_name": "Reka", "total_questions": "189"}, {"accuracy": "0.8790613422382673", "provider_name": "Relace", "total_questions": "1385"}, {"accuracy": "0.8905610702341137", "provider_name": "Sail Research", "total_questions": "897"}, {"accuracy": "0.9042808571428572", "provider_name": "SiliconFlow", "total_questions": "1386"}, {"accuracy": "0.9054832857142857", "provider_name": "StreamLake", "total_questions": "1386"}, {"accuracy": "0.8838381666666666", "provider_name": "Together", "total_questions": "1188"}, {"accuracy": "0.8742985", "provider_name": "Venice", "total_questions": "1188"}, {"accuracy": "0.8999134797927462", "provider_name": "Wafer", "total_questions": "1930"}], "rule": "median over providers (35 of 35 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-10-13", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.429, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4.1-flash_max", "source_row": {"Best score (across scorers)": "0.429", "Country": "China", "Model version": "deepseek-v4.1-flash_max", "Organization": "DeepSeek", "Release date": "2026-09-09", "Started at": "2026-10-13T12:59:33.000Z", "Training compute (FLOP)": "4.32e+24", "id": "3JJ4ChNxMo8CGAh3HrKNrw", "mean_score": "0.429", "stderr": "0.015658997547870177"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "deepseek-v4-1-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.745, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4.1-flash-20260910", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4.1-flash-20260910", "rows": [{"accuracy": "0.7677776666666667", "provider_name": "Alibaba", "total_questions": "300"}, {"accuracy": "0.7422221666666666", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.7826580270270269", "provider_name": "Baidu", "total_questions": "148"}, {"accuracy": "0.7284264213197968", "provider_name": "BaseTen", "total_questions": "197"}, {"accuracy": "0.713333", "provider_name": "Cloudflare", "total_questions": "50"}, {"accuracy": "0.742222", "provider_name": "CoreWeave", "total_questions": "150"}, {"accuracy": "0.7651005838926175", "provider_name": "Crusoe", "total_questions": "149"}, {"accuracy": "0.7644443333333334", "provider_name": "Decart", "total_questions": "150"}, {"accuracy": "0.7475", "provider_name": "DeepInfra", "total_questions": "400"}, {"accuracy": "0.76", "provider_name": "DekaLLM", "total_questions": "150"}, {"accuracy": "0.7644443333333334", "provider_name": "DigitalOcean", "total_questions": "300"}, {"accuracy": "0.7458124924623115", "provider_name": "Fireworks", "total_questions": "398"}, {"accuracy": "0.7571428571428571", "provider_name": "GMICloud", "total_questions": "350"}, {"accuracy": "0.7450001666666666", "provider_name": "InferenceNet", "total_questions": "300"}, {"accuracy": "0.7222223333333333", "provider_name": "Io Net", "total_questions": "150"}, {"accuracy": "0.745", "provider_name": "Ionstream", "total_questions": "200"}, {"accuracy": "0.6455556666666666", "provider_name": "Krea", "total_questions": "150"}, {"accuracy": "0.7416668333333333", "provider_name": "Makora", "total_questions": "300"}, {"accuracy": "0.7311110000000001", "provider_name": "Modal", "total_questions": "300"}, {"accuracy": "0.735000125", "provider_name": "Morph", "total_questions": "400"}, {"accuracy": "0.76", "provider_name": "Nebius", "total_questions": "100"}, {"accuracy": "0.7573333999999999", "provider_name": "NextBit", "total_questions": "250"}, {"accuracy": "0.764846632183908", "provider_name": "Novita", "total_questions": "348"}, {"accuracy": "0.7121210774410774", "provider_name": "OpenInference", "total_questions": "297"}, {"accuracy": "0.7305553333333333", "provider_name": "Parasail", "total_questions": "300"}, {"accuracy": "0.7433333333333333", "provider_name": "Phala", "total_questions": "300"}, {"accuracy": "0.755", "provider_name": "PrimeIntellect", "total_questions": "100"}, {"accuracy": "0.7666669999999999", "provider_name": "Reka", "total_questions": "50"}, {"accuracy": "0.732857", "provider_name": "Relace", "total_questions": "350"}, {"accuracy": "0.7550001666666666", "provider_name": "Sail Research", "total_questions": "300"}, {"accuracy": "0.7683335", "provider_name": "SiliconFlow", "total_questions": "200"}, {"accuracy": "0.7438097142857142", "provider_name": "StreamLake", "total_questions": "350"}, {"accuracy": "0.7311110000000001", "provider_name": "Together", "total_questions": "300"}, {"accuracy": "0.7322221666666667", "provider_name": "Venice", "total_questions": "300"}, {"accuracy": "0.7437710606060606", "provider_name": "Wafer", "total_questions": "495"}], "rule": "median over providers (35 of 35 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro-0813_max", "source_row": {"Best score (across scorers)": "0.46", "Country": "China", "Model version": "deepseek-v4-pro-0813_max", "Organization": "DeepSeek", "Release date": "2026-08-13", "Started at": "2026-10-13T13:13:34.000Z", "Training compute (FLOP)": "9.702e+24", "id": "C56VHYLMeww2wXebeJh3gg", "mean_score": "0.46", "stderr": "0.05009082659620331"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-19", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.43, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro-0813_max", "source_row": {"Best score (across scorers)": "0.43", "Country": "China", "Model version": "deepseek-v4-pro-0813_max", "Organization": "DeepSeek", "Release date": "2026-08-13", "Started at": "2026-08-19T18:14:49.000Z", "Training compute (FLOP)": "9.702e+24", "id": "c2KVkMCWnpo6BbnMWgdaDH", "mean_score": "0.43", "stderr": "0.0497569851956243"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-19", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2682926829268293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro-0813_max", "source_row": {"Best score (across scorers)": "0.2682926829268293", "Country": "China", "Model version": "deepseek-v4-pro-0813_max", "Organization": "DeepSeek", "Release date": "2026-08-13", "Started at": "2026-08-19T13:35:48.000Z", "Training compute (FLOP)": "9.702e+24", "id": "a4uV8HMMBAcHXLVSmZfBgX", "mean_score": "0.2682926829268293", "stderr": "0.07005564203095158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-19", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6456140350877193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro-0813_max", "source_row": {"Best score (across scorers)": "0.6456140350877193", "Country": "China", "Model version": "deepseek-v4-pro-0813_max", "Organization": "DeepSeek", "Release date": "2026-08-13", "Started at": "2026-08-19T13:35:48.000Z", "Training compute (FLOP)": "9.702e+24", "id": "b6yWmeFgsWeEiG9Qzs3NSf", "mean_score": "0.6456140350877193", "stderr": "0.028383475205435625"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-18", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9861111111111112, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro-0813_max", "source_row": {"Best score (across scorers)": "0.9861111111111112", "Country": "China", "Model version": "deepseek-v4-pro-0813_max", "Organization": "DeepSeek", "Release date": "2026-08-13", "Started at": "2026-08-18T21:03:23.000Z", "Training compute (FLOP)": "9.702e+24", "id": "4HVvM9ZWKKPLQPcC8WYYMT", "mean_score": "0.9861111111111112", "stderr": "0.00990980650017605"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-18", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9166666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro-0813_max", "source_row": {"Best score (across scorers)": "0.9166666666666666", "Country": "China", "Model version": "deepseek-v4-pro-0813_max", "Organization": "DeepSeek", "Release date": "2026-08-13", "Started at": "2026-08-18T21:03:23.000Z", "Training compute (FLOP)": "9.702e+24", "id": "nEJGtXGN588zynPkynZkxz", "mean_score": "0.9166666666666666", "stderr": "0.015242573629934338"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-09-19", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.892567903763697, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-pro-20260813", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "rows": [{"accuracy": "0.9093497692307693", "provider_name": "Alibaba", "run_count": "13", "total_questions": "2574"}, {"accuracy": "0.9121771666666666", "provider_name": "AtlasCloud", "run_count": "6", "total_questions": "1188"}, {"accuracy": "0.915614375", "provider_name": "Baidu", "run_count": "8", "total_questions": "1584"}, {"accuracy": "0.892567903763697", "provider_name": "BaseTen", "run_count": "11", "total_questions": "2099"}, {"accuracy": "0.8952784126849893", "provider_name": "Cloudflare", "run_count": "12", "total_questions": "2365"}, {"accuracy": "0.8819863000000001", "provider_name": "CoreWeave", "run_count": "10", "total_questions": "1980"}, {"accuracy": "0.8547321572398191", "provider_name": "DigitalOcean", "run_count": "9", "total_questions": "1768"}, {"accuracy": "0.882767", "provider_name": "GMICloud", "run_count": "11", "total_questions": "2178"}, {"accuracy": "0.8855217999999999", "provider_name": "Ionstream", "run_count": "5", "total_questions": "990"}, {"accuracy": "0.863636", "provider_name": "Nebius", "run_count": "1", "total_questions": "198"}, {"accuracy": "0.918981375", "provider_name": "NextBit", "run_count": "8", "total_questions": "1584"}, {"accuracy": "0.898697092413793", "provider_name": "Novita", "run_count": "11", "total_questions": "2175"}, {"accuracy": "0.8914141", "provider_name": "Parasail", "run_count": "10", "total_questions": "1980"}, {"accuracy": "0.9090909999999999", "provider_name": "Reka", "run_count": "1", "total_questions": "198"}, {"accuracy": "0.8878433783257624", "provider_name": "Sail Research", "run_count": "8", "total_questions": "1541"}, {"accuracy": "0.9060298181818183", "provider_name": "SiliconFlow", "run_count": "11", "total_questions": "2178"}, {"accuracy": "0.9046852723932015", "provider_name": "StreamLake", "run_count": "11", "total_questions": "2177"}, {"accuracy": "0.8732727052578362", "provider_name": "Together", "run_count": "10", "total_questions": "1978"}, {"accuracy": "0.8254771666666667", "provider_name": "Venice", "run_count": "6", "total_questions": "1188"}, {"accuracy": "0.9189512857142857", "provider_name": "Wafer", "run_count": "7", "total_questions": "1386"}], "rule": "median over providers (20 of 20 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5290581162324649, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro-0813_max", "source_row": {"Best score (across scorers)": "0.5290581162324649", "Country": "China", "Model version": "deepseek-v4-pro-0813_max", "Organization": "DeepSeek", "Release date": "2026-08-13", "Started at": "2026-08-27T19:30:20.000Z", "Training compute (FLOP)": "9.702e+24", "id": "2xfFf6KZAp2gUWYhHGkFgs", "mean_score": "0.5290581162324649", "stderr": "0.015808394665706597"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "deepseek-v4-pro-0813", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.767500125, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-pro-20260813", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260813", "rows": [{"accuracy": "0.7830555", "provider_name": "Alibaba", "total_questions": "600"}, {"accuracy": "0.7788889999999999", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.7788888333333334", "provider_name": "Baidu", "total_questions": "300"}, {"accuracy": "0.75", "provider_name": "BaseTen", "total_questions": "100"}, {"accuracy": "0.7622101098901098", "provider_name": "Cloudflare", "total_questions": "546"}, {"accuracy": "0.75750025", "provider_name": "CoreWeave", "total_questions": "400"}, {"accuracy": "0.7860360641891891", "provider_name": "DeepInfra", "total_questions": "296"}, {"accuracy": "0.7933334999999999", "provider_name": "DekaLLM", "total_questions": "100"}, {"accuracy": "0.6724726434426229", "provider_name": "DigitalOcean", "total_questions": "488"}, {"accuracy": "0.7366665", "provider_name": "Fireworks", "total_questions": "100"}, {"accuracy": "0.7426667", "provider_name": "GMICloud", "total_questions": "500"}, {"accuracy": "0.72", "provider_name": "Io Net", "total_questions": "50"}, {"accuracy": "0.7525252929292929", "provider_name": "Ionstream", "total_questions": "297"}, {"accuracy": "0.706667", "provider_name": "Nebius", "total_questions": "50"}, {"accuracy": "0.7628571428571429", "provider_name": "NextBit", "total_questions": "350"}, {"accuracy": "0.7811111666666667", "provider_name": "Novita", "total_questions": "300"}, {"accuracy": "0.7707407777777777", "provider_name": "Parasail", "total_questions": "450"}, {"accuracy": "0.767500125", "provider_name": "Phala", "total_questions": "400"}, {"accuracy": "0.7733335", "provider_name": "Reka", "total_questions": "100"}, {"accuracy": "0.7755556666666668", "provider_name": "Relace", "total_questions": "150"}, {"accuracy": "0.7544342825688073", "provider_name": "Sail Research", "total_questions": "545"}, {"accuracy": "0.7806513735632183", "provider_name": "SiliconFlow", "total_questions": "348"}, {"accuracy": "0.7733332222222222", "provider_name": "StreamLake", "total_questions": "450"}, {"accuracy": "0.7588888888888888", "provider_name": "Together", "total_questions": "450"}, {"accuracy": "0.7677778333333333", "provider_name": "Venice", "total_questions": "300"}, {"accuracy": "0.7771429999999999", "provider_name": "Wafer", "total_questions": "350"}], "rule": "median over providers (26 of 26 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-10-15", "model": "muse-spark-open", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3170731707317073, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Muse-Spark-Open-816B-A42B_xhigh", "source_row": {"Best score (across scorers)": "0.3170731707317073", "Country": "United States of America", "Model version": "Muse-Spark-Open-816B-A42B_xhigh", "Organization": "Meta AI", "Release date": "2026-10-15", "Started at": "2026-10-15T18:52:04.000Z", "id": "YifYkfNddMSuE8uBNdbvLi", "mean_score": "0.3170731707317073", "stderr": "0.07357611282438223"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-10-15", "model": "muse-spark-open", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6982456140350877, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Muse-Spark-Open-816B-A42B_xhigh", "source_row": {"Best score (across scorers)": "0.6982456140350877", "Country": "United States of America", "Model version": "Muse-Spark-Open-816B-A42B_xhigh", "Organization": "Meta AI", "Release date": "2026-10-15", "Started at": "2026-10-15T18:52:04.000Z", "id": "e3e3h9r7Fouu4wsAeVzTnV", "mean_score": "0.6982456140350877", "stderr": "0.027237781633151586"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-10-15", "model": "muse-spark-open", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9805555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Muse-Spark-Open-816B-A42B_xhigh", "source_row": {"Best score (across scorers)": "0.9805555555555555", "Country": "United States of America", "Model version": "Muse-Spark-Open-816B-A42B_xhigh", "Organization": "Meta AI", "Release date": "2026-10-15", "Started at": "2026-10-15T18:52:04.000Z", "id": "hB4XzBdWkdpG3WJVYqeRM8", "mean_score": "0.9805555555555555", "stderr": "0.01253502388483925"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-10-15", "model": "muse-spark-open", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8857323232323232, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Muse-Spark-Open-816B-A42B_xhigh", "source_row": {"Best score (across scorers)": "0.8857323232323232", "Country": "United States of America", "Model version": "Muse-Spark-Open-816B-A42B_xhigh", "Organization": "Meta AI", "Release date": "2026-10-15", "Started at": "2026-10-15T18:52:04.000Z", "id": "JnKpor8c3EKWGvmyjLFVzH", "mean_score": "0.8857323232323232", "stderr": "0.018706209910489154"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "muse-spark-open", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.905724, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta/muse-spark-open-20261015", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta/muse-spark-open-20261015", "rows": [{"accuracy": "0.905724", "provider_name": "Fireworks", "total_questions": "198"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-10-15", "model": "muse-spark-open", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.566, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Muse-Spark-Open-816B-A42B_xhigh", "source_row": {"Best score (across scorers)": "0.566", "Country": "United States of America", "Model version": "Muse-Spark-Open-816B-A42B_xhigh", "Organization": "Meta AI", "Release date": "2026-10-15", "Started at": "2026-10-15T18:52:04.000Z", "id": "dNuxLdV4DWqfSvtTq6NnK8", "mean_score": "0.566", "stderr": "0.01568087656637504"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "muse-spark-open", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.78, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta/muse-spark-open-20261015", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta/muse-spark-open-20261015", "rows": [{"accuracy": "0.78", "provider_name": "Fireworks", "total_questions": "50"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_high", "source_row": {"Best score (across scorers)": "0.37", "Country": "China", "Model version": "kimi-k3_high", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-10-13T17:10:13.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "X2QNPHFXuWosA6AiuvVtDv", "mean_score": "0.37", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-11", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3416666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_max", "source_row": {"Best score (across scorers)": "0.3416666666666667", "Country": "China", "Model version": "kimi-k3_max", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-09-11T03:00:40.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "fMkb7c7AqFoehCqiy2tLR5", "mean_score": "0.3416666666666667", "stderr": "0.061169934025019475"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-29", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_max", "source_row": {"Best score (across scorers)": "0.26", "Country": "China", "Model version": "kimi-k3_max", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-07-29T03:26:08.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "eCq63CuekohAswYftWXJVj", "mean_score": "0.26", "stderr": "0.0440844002276808"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-07-17", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3902439024390244, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_max", "source_row": {"Best score (across scorers)": "0.3902439024390244", "Country": "China", "Model version": "kimi-k3_max", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-07-17T01:48:42.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "gZERSAQwxBAevQzBsWfUie", "mean_score": "0.3902439024390244", "stderr": "0.07712872341874095"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-07-17", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.721830985915493, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_max", "source_row": {"Best score (across scorers)": "0.721830985915493", "Country": "China", "Model version": "kimi-k3_max", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-07-17T01:48:42.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "bcntamvGsSH6QgETdoaSph", "mean_score": "0.721830985915493", "stderr": "0.026636607935112643"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-16", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9722222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_max", "source_row": {"Best score (across scorers)": "0.9722222222222222", "Country": "China", "Model version": "kimi-k3_max", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-07-16T21:59:40.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "A33om9MTYMC7xYeLv9X7Ha", "mean_score": "0.9722222222222222", "stderr": "0.011158359139033182"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-07-16", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9311868686868687, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_max", "source_row": {"Best score (across scorers)": "0.9311868686868687", "Country": "China", "Model version": "kimi-k3_max", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-07-16T21:59:40.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "B8MaPUe3EYwUZASsNqE7C9", "mean_score": "0.9311868686868687", "stderr": "0.014937225364194858"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9157357830756013, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k3-20260715", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k3-20260715", "rows": [{"accuracy": "0.9159637816307403", "provider_name": "AkashML", "total_questions": "1067"}, {"accuracy": "0.9212961249999999", "provider_name": "Alibaba", "total_questions": "1584"}, {"accuracy": "0.92312", "provider_name": "Amazon Bedrock", "total_questions": "594"}, {"accuracy": "0.9193136803377565", "provider_name": "BaseTen", "total_questions": "2487"}, {"accuracy": "0.910344342048864", "provider_name": "Chutes", "total_questions": "2333"}, {"accuracy": "0.924032", "provider_name": "Decart", "total_questions": "792"}, {"accuracy": "0.900973165044435", "provider_name": "DeepInfra", "total_questions": "2363"}, {"accuracy": "0.8788940912152681", "provider_name": "DigitalOcean", "total_questions": "3563"}, {"accuracy": "0.9264645740259738", "provider_name": "Fireworks", "total_questions": "5775"}, {"accuracy": "0.9196950955414013", "provider_name": "InferenceNet", "total_questions": "2669"}, {"accuracy": "0.9262129285099052", "provider_name": "Makora", "total_questions": "1161"}, {"accuracy": "0.9214533832285114", "provider_name": "Modal", "total_questions": "2385"}, {"accuracy": "0.9157357830756013", "provider_name": "Moonshot AI", "total_questions": "2328"}, {"accuracy": "0.9132906754665453", "provider_name": "Morph", "total_questions": "4394"}, {"accuracy": "0.9295867436026091", "provider_name": "Parasail", "total_questions": "1993"}, {"accuracy": "0.9067522535599448", "provider_name": "Phala", "total_questions": "2177"}, {"accuracy": "0.9262631428571431", "provider_name": "Relace", "total_questions": "1372"}, {"accuracy": "0.9200982307692309", "provider_name": "Sail Research", "total_questions": "2574"}, {"accuracy": "0.9049022275600507", "provider_name": "Together", "total_questions": "2373"}, {"accuracy": "0.8833967254326782", "provider_name": "Wafer", "total_questions": "3409"}], "rule": "first-party provider (1 of 20 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.506, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k3_max", "source_row": {"Best score (across scorers)": "0.506", "Country": "China", "Model version": "kimi-k3_max", "Organization": "Moonshot", "Release date": "2026-07-16", "Started at": "2026-08-27T19:36:12.000Z", "Training compute (FLOP)": "2.0001e+25", "id": "mZLWt2JSYvJyGhMvji7CT8", "mean_score": "0.506", "stderr": "0.015818160898606836"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "kimi-k3", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7347416579476861, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k3-20260715", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k3-20260715", "rows": [{"accuracy": "0.7649122421052632", "provider_name": "AkashML", "total_questions": "95"}, {"accuracy": "0.741666625", "provider_name": "Alibaba", "total_questions": "400"}, {"accuracy": "0.7533333333333333", "provider_name": "Amazon Bedrock", "total_questions": "150"}, {"accuracy": "0.7557079178082191", "provider_name": "BaseTen", "total_questions": "146"}, {"accuracy": "0.7583335", "provider_name": "Decart", "total_questions": "200"}, {"accuracy": "0.746666625", "provider_name": "DeepInfra", "total_questions": "400"}, {"accuracy": "0.7205554166666667", "provider_name": "DigitalOcean", "total_questions": "600"}, {"accuracy": "0.7198066588628762", "provider_name": "Fireworks", "total_questions": "897"}, {"accuracy": "0.7609090909090909", "provider_name": "InferenceNet", "total_questions": "550"}, {"accuracy": "0.7441076363636364", "provider_name": "Makora", "total_questions": "198"}, {"accuracy": "0.7362964444444444", "provider_name": "Modal", "total_questions": "450"}, {"accuracy": "0.7347416579476861", "provider_name": "Moonshot AI", "total_questions": "497"}, {"accuracy": "0.7428571904761905", "provider_name": "Morph", "total_questions": "1050"}, {"accuracy": "0.7361904285714286", "provider_name": "Parasail", "total_questions": "350"}, {"accuracy": "0.7370185075376885", "provider_name": "Phala", "total_questions": "398"}, {"accuracy": "0.7392550315186246", "provider_name": "Relace", "total_questions": "349"}, {"accuracy": "0.7313334", "provider_name": "Sail Research", "total_questions": "500"}, {"accuracy": "0.731515090909091", "provider_name": "Together", "total_questions": "550"}, {"accuracy": "0.7569232307692307", "provider_name": "Wafer", "total_questions": "650"}], "rule": "first-party provider (1 of 19 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.41, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_max", "source_row": {"Best score (across scorers)": "0.41", "Country": "United States of America", "Model version": "claude-opus-4-7_max", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-10-13T13:36:23.000Z", "id": "XXPqVNR7WCFFA2Rnzjcbk6", "mean_score": "0.41", "stderr": "0.04943110704237103"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_max", "source_row": {"Best score (across scorers)": "0.3333333333333333", "Country": "United States of America", "Model version": "claude-opus-4-7_max", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-09-10T20:23:16.000Z", "id": "NPkiPtHFjjYvdXoG2LLg7A", "mean_score": "0.3333333333333333", "stderr": "0.06137164119322156"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-25", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.28, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_max", "source_row": {"Best score (across scorers)": "0.28", "Country": "United States of America", "Model version": "claude-opus-4-7_max", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-07-25T01:43:24.000Z", "id": "VFYhE54autgdYcMJNXicwU", "mean_score": "0.28", "stderr": "0.04512608598542129"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-10", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3170731707317073, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_max", "source_row": {"Best score (across scorers)": "0.3170731707317073", "Country": "United States of America", "Model version": "claude-opus-4-7_max", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-06-10T21:41:54.000Z", "id": "P4LXQ9PyDDziEANMB5jVRd", "mean_score": "0.3170731707317073", "stderr": "0.07357611282438223"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-10", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7017543859649122, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_max", "source_row": {"Best score (across scorers)": "0.7017543859649122", "Country": "United States of America", "Model version": "claude-opus-4-7_max", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-06-10T21:41:54.000Z", "id": "ZkNQ7GER7Mc9FSPmpUzgKH", "mean_score": "0.7017543859649122", "stderr": "0.027146911721220194"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-04-17", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.978, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_xhigh", "source_row": {"Best score (across scorers)": "0.978", "Country": "United States of America", "Model version": "claude-opus-4-7_xhigh", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-04-17T18:26:45.000Z", "id": "jyCZnpXG2bh9G43X8pT75m", "mean_score": "0.978", "stderr": "0.022"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-04-20", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8347107438016529, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_max", "source_row": {"Best score (across scorers)": "0.8347107438016529", "Country": "United States of America", "Model version": "claude-opus-4-7_max", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-04-20T13:16:09.316Z", "id": "nCmGWWMBip2s9AVEZNYXDX", "mean_score": "0.8347107438016529", "stderr": "0.016901169179529604"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-04-17", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9015151515151515, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_xhigh", "source_row": {"Best score (across scorers)": "0.9015151515151515", "Country": "United States of America", "Model version": "claude-opus-4-7_xhigh", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-04-17T16:16:45.000Z", "id": "VJezHrJ5YPCbpui8pZgcJg", "mean_score": "0.9015151515151515", "stderr": "0.018464275887293338"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.892375117408907, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.7-opus-20260416", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "rows": [{"accuracy": "0.8909229583333332", "provider_name": "Amazon Bedrock", "total_questions": "4752"}, {"accuracy": "0.892375117408907", "provider_name": "Anthropic", "total_questions": "2964"}, {"accuracy": "0.8895921012658227", "provider_name": "Azure", "total_questions": "2370"}, {"accuracy": "0.8988212999999999", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.8951083888888889", "provider_name": "Google", "total_questions": "7128"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.517, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-7_xhigh", "source_row": {"Best score (across scorers)": "0.517", "Country": "United States of America", "Model version": "claude-opus-4-7_xhigh", "Organization": "Anthropic", "Release date": "2026-04-16", "Started at": "2026-08-27T19:27:26.000Z", "id": "2c4rvmf5N7QXTugrFLMZ83", "mean_score": "0.517", "stderr": "0.015810153729833274"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-opus-4-7", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7757142142857143, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.7-opus-20260416", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.7-opus-20260416", "rows": [{"accuracy": "0.7748484090909091", "provider_name": "Amazon Bedrock", "total_questions": "1100"}, {"accuracy": "0.7757142142857143", "provider_name": "Anthropic", "total_questions": "700"}, {"accuracy": "0.7727271818181819", "provider_name": "Azure", "total_questions": "550"}, {"accuracy": "0.7681481111111111", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7660606363636364", "provider_name": "Google", "total_questions": "1650"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.51, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview_high", "source_row": {"Best score (across scorers)": "0.51", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview_high", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-10-13T13:34:13.000Z", "id": "FTDFCpXzDTgfgWBNdHLHGc", "mean_score": "0.51", "stderr": "0.05024183937956913"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26666666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview_high", "source_row": {"Best score (across scorers)": "0.26666666666666666", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview_high", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-09-10T20:23:45.000Z", "id": "NzNcASjLtNrngbNQKcD3G3", "mean_score": "0.26666666666666666", "stderr": "0.056331678510110804"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-27", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview_high", "source_row": {"Best score (across scorers)": "0.34", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview_high", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-07-27T18:29:16.000Z", "id": "g4bbCaiFDqw2YYNDAcmn2a", "mean_score": "0.34", "stderr": "0.04760952285695234"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2682926829268293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview", "source_row": {"Best score (across scorers)": "0.2682926829268293", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-06-11T23:35:57.000Z", "id": "eXVWr3RBeSsWy5DAU8ed46", "mean_score": "0.2682926829268293", "stderr": "0.07005564203095158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5964912280701754, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview", "source_row": {"Best score (across scorers)": "0.5964912280701754", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-06-11T23:35:57.000Z", "id": "hVifTUdQZrFEYabgCRu8Qq", "mean_score": "0.5964912280701754", "stderr": "0.02911181956525014"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-02-20", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.956, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview", "source_row": {"Best score (across scorers)": "0.956", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-02-20T04:39:36.998Z", "id": "bXgZZgzRE3wXoSRUgrbsVf", "mean_score": "0.956", "stderr": "0.031"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-24", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.756198347107438, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview-customtools", "source_row": {"Best score (across scorers)": "0.756198347107438", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview-customtools", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-02-24T13:34:48.126Z", "id": "8QQQWDgmmEsmQVUJWcxx4P", "mean_score": "0.756198347107438", "stderr": "0.019537213360116365"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9444444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview_high", "source_row": {"Best score (across scorers)": "0.9444444444444444", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview_high", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-08-06T22:30:37.000Z", "id": "BgmrBFsbsaD8rNpebv49Da", "mean_score": "0.9444444444444444", "stderr": "0.016319950700767385"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.9483723333333332, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.1-pro-preview-20260219", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "rows": [{"accuracy": "0.95202", "provider_name": "Google", "total_questions": "396"}, {"accuracy": "0.9483723333333332", "provider_name": "Google AI Studio", "total_questions": "594"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.735, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-pro-preview_high", "source_row": {"Best score (across scorers)": "0.735", "Country": "United States of America", "Model version": "gemini-3.1-pro-preview_high", "Organization": "Google DeepMind", "Release date": "2026-02-19", "Started at": "2026-08-10T21:35:39.000Z", "id": "VGeF8Xac4WcSAGgSXVdEBi", "mean_score": "0.735", "stderr": "0.013963164754810065"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-3-1-pro", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7422223333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.1-pro-preview-20260219", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.1-pro-preview-20260219", "rows": [{"accuracy": "0.7666666666666667", "provider_name": "Google", "total_questions": "150"}, {"accuracy": "0.7422223333333333", "provider_name": "Google AI Studio", "total_questions": "150"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.45, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.45", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-10-13T13:33:53.000Z", "id": "XUDdWKUZBhdWPPQBRgx8Pw", "mean_score": "0.45", "stderr": "0.05"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-27", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.32, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.32", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-07-27T18:28:52.000Z", "id": "GQKsxFbdBWB546zE8ci49f", "mean_score": "0.32", "stderr": "0.04688261722621506"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-10", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2682926829268293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.2682926829268293", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-06-10T17:32:38.000Z", "id": "cXn5zzns6VFwbTgotDwZjV", "mean_score": "0.2682926829268293", "stderr": "0.07005564203095158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-10", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6280701754385964, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.6280701754385964", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-06-10T17:32:38.000Z", "id": "72DQVwbYiQW5pjhshjBaGU", "mean_score": "0.6280701754385964", "stderr": "0.028679753751074618"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-05-25", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.9555555555555556", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-05-25T13:10:18.000Z", "id": "eYmgjYJfM7Utm46nfoRfZ6", "mean_score": "0.9555555555555556", "stderr": "0.026696111689702044"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-06-01", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7933884297520661, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.7933884297520661", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-06-01T10:22:55.131Z", "id": "dwRzFGTghYEy4HNZajJXoc", "mean_score": "0.7933884297520661", "stderr": "0.018422420250424888"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-05-22", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.928030303030303, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.928030303030303", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-05-22T16:02:44.000Z", "id": "dYYtcwENFRtm9jZTDA2p9C", "mean_score": "0.928030303030303", "stderr": "0.016387002835275905"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.9242421666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.5-flash-20260519", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-20260519", "rows": [{"accuracy": "0.9242421666666667", "provider_name": "Google", "total_questions": "1188"}, {"accuracy": "0.925926", "provider_name": "Google AI Studio", "total_questions": "594"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.662, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash_high", "source_row": {"Best score (across scorers)": "0.662", "Country": "United States of America", "Model version": "gemini-3.5-flash_high", "Organization": "Google DeepMind", "Release date": "2026-05-19", "Started at": "2026-08-27T19:30:35.000Z", "id": "GJ6YsSuzyRD9YXe6SBwvsu", "mean_score": "0.662", "stderr": "0.014965960710224383"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-3-5-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7333336666666668, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.5-flash-20260519", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-20260519", "rows": [{"accuracy": "0.7488737702702704", "provider_name": "Google", "total_questions": "296"}, {"accuracy": "0.7333336666666668", "provider_name": "Google AI Studio", "total_questions": "150"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.33, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_xhigh", "source_row": {"Best score (across scorers)": "0.33", "Country": "United States of America", "Model version": "grok-4.6_xhigh", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-10-13T13:14:03.000Z", "id": "NiuExVccWkWbWqi6ex6vvf", "mean_score": "0.33", "stderr": "0.04725815626252608"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-24", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_xhigh", "source_row": {"Best score (across scorers)": "0.4", "Country": "United States of America", "Model version": "grok-4.6_xhigh", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-09-24T17:32:49.000Z", "id": "GM5e6UBbgeBJ83NJe6R6y9", "mean_score": "0.4", "stderr": "0.06266221885886569"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-14", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_xhigh", "source_row": {"Best score (across scorers)": "0.34", "Country": "United States of America", "Model version": "grok-4.6_xhigh", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-08-14T20:27:46.000Z", "id": "9cXtChGrowgGDwa9AcAecB", "mean_score": "0.34", "stderr": "0.04760952285695234"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-14", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3170731707317073, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_xhigh", "source_row": {"Best score (across scorers)": "0.3170731707317073", "Country": "United States of America", "Model version": "grok-4.6_xhigh", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-08-14T15:46:50.000Z", "id": "bg3JRdd33iWfvRoGSvyXXE", "mean_score": "0.3170731707317073", "stderr": "0.07357611282438223"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-14", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6596491228070176, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_xhigh", "source_row": {"Best score (across scorers)": "0.6596491228070176", "Country": "United States of America", "Model version": "grok-4.6_xhigh", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-08-14T15:46:50.000Z", "id": "Zb2AKwa942D7dBTTcebAba", "mean_score": "0.6596491228070176", "stderr": "0.028116467881852295"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-14", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9916666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_xhigh", "source_row": {"Best score (across scorers)": "0.9916666666666667", "Country": "United States of America", "Model version": "grok-4.6_xhigh", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-08-14T13:47:42.000Z", "id": "7crLvnfQc987dC6mr3iGbf", "mean_score": "0.9916666666666667", "stderr": "0.004700633956814717"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-12", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9400252525252525, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_high", "source_row": {"Best score (across scorers)": "0.9400252525252525", "Country": "United States of America", "Model version": "grok-4.6_high", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-08-12T19:39:52.000Z", "id": "hnP6Z88x8bPEBRZ2PW6kps", "mean_score": "0.9400252525252525", "stderr": "0.014480018812911388"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9246375695652174, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "x-ai/grok-4.6-20260810", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "x-ai/grok-4.6-20260810", "rows": [{"accuracy": "0.903508802631579", "provider_name": "Amazon Bedrock", "total_questions": "456"}, {"accuracy": "0.9246375695652174", "provider_name": "xAI", "total_questions": "1380"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.493, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.6_high", "source_row": {"Best score (across scorers)": "0.493", "Country": "United States of America", "Model version": "grok-4.6_high", "Organization": "xAI", "Release date": "2026-08-12", "Started at": "2026-08-27T19:29:53.000Z", "id": "TpHYtuPG6kYZFvMK7Z6Hqj", "mean_score": "0.493", "stderr": "0.01581774956184353"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "grok-4-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.747619, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "x-ai/grok-4.6-20260810", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "x-ai/grok-4.6-20260810", "rows": [{"accuracy": "0.7433335", "provider_name": "Amazon Bedrock", "total_questions": "100"}, {"accuracy": "0.747619", "provider_name": "xAI", "total_questions": "350"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.45, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5_max", "source_row": {"Best score (across scorers)": "0.45", "Country": "United States of America", "Model version": "claude-sonnet-5_max", "Organization": "Anthropic", "Release date": "2026-06-30", "Started at": "2026-10-13T13:23:26.000Z", "id": "PN9exWpeJGXJB8tZGX32e4", "mean_score": "0.45", "stderr": "0.05"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-28", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.35, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5_max", "source_row": {"Best score (across scorers)": "0.35", "Country": "United States of America", "Model version": "claude-sonnet-5_max", "Organization": "Anthropic", "Release date": "2026-06-30", "Started at": "2026-07-28T21:09:50.000Z", "id": "YdcyYXLzqfPUSSgGv4MTED", "mean_score": "0.35", "stderr": "0.04793724854411023"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-30", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2926829268292683, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5_max", "source_row": {"Best score (across scorers)": "0.2926829268292683", "Country": "United States of America", "Model version": "claude-sonnet-5_max", "Organization": "Anthropic", "Release date": "2026-06-30", "Started at": "2026-06-30T21:54:08.000Z", "id": "bqiVinHXFcgyFGPstJwXJN", "mean_score": "0.2926829268292683", "stderr": "0.07194088392074452"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-30", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.656140350877193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5_max", "source_row": {"Best score (across scorers)": "0.656140350877193", "Country": "United States of America", "Model version": "claude-sonnet-5_max", "Organization": "Anthropic", "Release date": "2026-06-30", "Started at": "2026-06-30T21:54:08.000Z", "id": "V6YuGpcu8MCbJNXKoJMA6b", "mean_score": "0.656140350877193", "stderr": "0.02818576398907258"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-01", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9472222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5_xhigh", "source_row": {"Best score (across scorers)": "0.9472222222222222", "Country": "United States of America", "Model version": "claude-sonnet-5_xhigh", "Organization": "Anthropic", "Release date": "2026-06-30", "Started at": "2026-07-01T16:06:23.000Z", "id": "T2wFfuSkUdyjpyqurxNmNK", "mean_score": "0.9472222222222222", "stderr": "0.022644349282618412"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-07-01", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9053030303030303, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5_xhigh", "source_row": {"Best score (across scorers)": "0.9053030303030303", "Country": "United States of America", "Model version": "claude-sonnet-5_xhigh", "Organization": "Anthropic", "Release date": "2026-06-30", "Started at": "2026-07-01T01:22:14.000Z", "id": "HhiYxEx3Bf8YDPUh34K4DC", "mean_score": "0.9053030303030303", "stderr": "0.01783832131341021"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8576035833333334, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-sonnet-5-20260630", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "rows": [{"accuracy": "0.8589818529411762", "provider_name": "Amazon Bedrock", "total_questions": "6732"}, {"accuracy": "0.8576035833333334", "provider_name": "Anthropic", "total_questions": "2376"}, {"accuracy": "0.8617332608695653", "provider_name": "Azure", "total_questions": "4554"}, {"accuracy": "0.8405724", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.8499179397894735", "provider_name": "Google", "total_questions": "7125"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.337, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-5_max", "source_row": {"Best score (across scorers)": "0.337", "Country": "United States of America", "Model version": "claude-sonnet-5_max", "Organization": "Anthropic", "Release date": "2026-06-30", "Started at": "2026-08-27T19:27:42.000Z", "id": "LPo74T9L3CawnKX5LLUrSZ", "mean_score": "0.337", "stderr": "0.014955087918653461"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-sonnet-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.76, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-sonnet-5-20260630", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-sonnet-5-20260630", "rows": [{"accuracy": "0.7597916562500001", "provider_name": "Amazon Bedrock", "total_questions": "1600"}, {"accuracy": "0.76", "provider_name": "Anthropic", "total_questions": "550"}, {"accuracy": "0.7546032380952381", "provider_name": "Azure", "total_questions": "1050"}, {"accuracy": "0.7511112222222222", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7565656363636365", "provider_name": "Google", "total_questions": "1650"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3_max", "source_row": {"Best score (across scorers)": "0.37", "Country": "China", "Model version": "glm-5.3_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-14", "Started at": "2026-10-13T21:22:18.000Z", "id": "kG3hbd8qLEd6fLAJHyQTiN", "mean_score": "0.37", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-30", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.33, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3_max", "source_row": {"Best score (across scorers)": "0.33", "Country": "China", "Model version": "glm-5.3_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-14", "Started at": "2026-08-30T23:33:33.000Z", "id": "UUq99Qhq3qizcoJo5vdhuK", "mean_score": "0.33", "stderr": "0.04725815626252608"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-25", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2926829268292683, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3_max", "source_row": {"Best score (across scorers)": "0.2926829268292683", "Country": "China", "Model version": "glm-5.3_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-14", "Started at": "2026-08-25T14:20:20.000Z", "id": "4S3ZrJDkwh4gmGrmJQdvzR", "mean_score": "0.2926829268292683", "stderr": "0.07194088392074452"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-25", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6877192982456141, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3_max", "source_row": {"Best score (across scorers)": "0.6877192982456141", "Country": "China", "Model version": "glm-5.3_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-14", "Started at": "2026-08-25T14:20:20.000Z", "id": "iXvqUzwxAiA3Rg3dwnutFs", "mean_score": "0.6877192982456141", "stderr": "0.027499133473298815"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-24", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3_max", "source_row": {"Best score (across scorers)": "0.9111111111111111", "Country": "China", "Model version": "glm-5.3_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-14", "Started at": "2026-08-24T19:44:18.000Z", "id": "goZR5yGxZic9TZ23ezCL3w", "mean_score": "0.9111111111111111", "stderr": "0.03454754040013868"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-24", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9090909090909091, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3_max", "source_row": {"Best score (across scorers)": "0.9090909090909091", "Country": "China", "Model version": "glm-5.3_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-14", "Started at": "2026-08-24T19:44:18.000Z", "id": "dLDBahYBTGRNjbsXA5pyr3", "mean_score": "0.9090909090909091", "stderr": "0.015851578202142853"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8639263412162163, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.3-20260816", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-20260816", "rows": [{"accuracy": "0.8332576446056212", "provider_name": "AkashML", "total_questions": "2206"}, {"accuracy": "0.836088", "provider_name": "Alibaba", "total_questions": "2178"}, {"accuracy": "0.857416679028133", "provider_name": "AtlasCloud", "total_questions": "1564"}, {"accuracy": "0.8522533600867679", "provider_name": "Baidu", "total_questions": "1383"}, {"accuracy": "0.8515657312005884", "provider_name": "BaseTen", "total_questions": "5439"}, {"accuracy": "0.8245299743589743", "provider_name": "Cloudflare", "total_questions": "1950"}, {"accuracy": "0.8236842684210525", "provider_name": "CoreWeave", "total_questions": "380"}, {"accuracy": "0.8453582857142857", "provider_name": "Crusoe", "total_questions": "1386"}, {"accuracy": "0.8449956870229007", "provider_name": "Decart", "total_questions": "1572"}, {"accuracy": "0.870157356321839", "provider_name": "DeepInfra", "total_questions": "1566"}, {"accuracy": "0.8489013956043955", "provider_name": "DekaLLM", "total_questions": "364"}, {"accuracy": "0.8809836805006587", "provider_name": "DigitalOcean", "total_questions": "1518"}, {"accuracy": "0.8481498894762135", "provider_name": "Fireworks", "total_questions": "2081"}, {"accuracy": "0.8273474444444444", "provider_name": "Friendli", "total_questions": "1782"}, {"accuracy": "0.8453046666666667", "provider_name": "GMICloud", "total_questions": "1782"}, {"accuracy": "0.8591320634715027", "provider_name": "Inceptron", "total_questions": "1544"}, {"accuracy": "0.8580637958148383", "provider_name": "InferenceNet", "total_questions": "1577"}, {"accuracy": "0.8223551976047904", "provider_name": "Io Net", "total_questions": "1169"}, {"accuracy": "0.8384655804464972", "provider_name": "Makora", "total_questions": "1299"}, {"accuracy": "0.8458659444444445", "provider_name": "Mistral", "total_questions": "3564"}, {"accuracy": "0.8356714444444444", "provider_name": "Modal", "total_questions": "1782"}, {"accuracy": "0.8488849929577466", "provider_name": "ModelRun", "total_questions": "568"}, {"accuracy": "0.8249891001283697", "provider_name": "Morph", "total_questions": "2337"}, {"accuracy": "0.8358585", "provider_name": "Nebius", "total_questions": "792"}, {"accuracy": "0.8357648888888889", "provider_name": "Novita", "total_questions": "1782"}, {"accuracy": "0.8304393877813504", "provider_name": "Parasail", "total_questions": "1555"}, {"accuracy": "0.848859786319218", "provider_name": "Phala", "total_questions": "1535"}, {"accuracy": "0.8430333386243387", "provider_name": "PrimeIntellect", "total_questions": "1323"}, {"accuracy": "0.8401923283835383", "provider_name": "Reka", "total_questions": "2357"}, {"accuracy": "0.8450553324808183", "provider_name": "Relace", "total_questions": "782"}, {"accuracy": "0.8434823631462101", "provider_name": "Sail Research", "total_questions": "3153"}, {"accuracy": "0.8466410969581748", "provider_name": "SiliconFlow", "total_questions": "1578"}, {"accuracy": "0.819865", "provider_name": "StreamLake", "total_questions": "198"}, {"accuracy": "0.8112779399727148", "provider_name": "Tenstorrent", "total_questions": "733"}, {"accuracy": "0.8421298152723189", "provider_name": "Together", "total_questions": "1781"}, {"accuracy": "0.8634435114503817", "provider_name": "Venice", "total_questions": "1572"}, {"accuracy": "0.8573675040431266", "provider_name": "Wafer", "total_questions": "2968"}, {"accuracy": "0.8639263412162163", "provider_name": "Z.AI", "total_questions": "1776"}], "rule": "first-party provider (1 of 38 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-28", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.41, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3_max", "source_row": {"Best score (across scorers)": "0.41", "Country": "China", "Model version": "glm-5.3_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-14", "Started at": "2026-08-28T17:00:06.000Z", "id": "TY8R8FV7zCwByEipkqyg2P", "mean_score": "0.41", "stderr": "0.015560917136921659"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "glm-5-3", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.78083325, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.3-20260816", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-20260816", "rows": [{"accuracy": "0.7382865983935742", "provider_name": "AkashML", "total_questions": "249"}, {"accuracy": "0.7769696363636364", "provider_name": "Alibaba", "total_questions": "550"}, {"accuracy": "0.7777778333333333", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.7653332", "provider_name": "Baidu", "total_questions": "250"}, {"accuracy": "0.7836751845597104", "provider_name": "BaseTen", "total_questions": "829"}, {"accuracy": "0.7772191809045226", "provider_name": "Cloudflare", "total_questions": "398"}, {"accuracy": "0.7755098163265306", "provider_name": "CoreWeave", "total_questions": "98"}, {"accuracy": "0.7844445", "provider_name": "Crusoe", "total_questions": "300"}, {"accuracy": "0.7714285714285715", "provider_name": "Decart", "total_questions": "350"}, {"accuracy": "0.7711111666666667", "provider_name": "DeepInfra", "total_questions": "300"}, {"accuracy": "0.805", "provider_name": "DekaLLM", "total_questions": "100"}, {"accuracy": "0.7779999000000001", "provider_name": "DigitalOcean", "total_questions": "500"}, {"accuracy": "0.788", "provider_name": "Fireworks", "total_questions": "500"}, {"accuracy": "0.77916675", "provider_name": "Friendli", "total_questions": "400"}, {"accuracy": "0.7575", "provider_name": "GMICloud", "total_questions": "400"}, {"accuracy": "0.7685714285714286", "provider_name": "Inceptron", "total_questions": "350"}, {"accuracy": "0.7616665", "provider_name": "InferenceNet", "total_questions": "400"}, {"accuracy": "0.7799998", "provider_name": "Io Net", "total_questions": "250"}, {"accuracy": "0.7722221666666667", "provider_name": "Makora", "total_questions": "300"}, {"accuracy": "0.7729629444444445", "provider_name": "Mistral", "total_questions": "900"}, {"accuracy": "0.768333375", "provider_name": "Modal", "total_questions": "400"}, {"accuracy": "0.7833334999999999", "provider_name": "ModelRun", "total_questions": "100"}, {"accuracy": "0.7611212248628885", "provider_name": "Morph", "total_questions": "547"}, {"accuracy": "0.7855944623115578", "provider_name": "Nebius", "total_questions": "199"}, {"accuracy": "0.76916675", "provider_name": "Novita", "total_questions": "400"}, {"accuracy": "0.775238", "provider_name": "Parasail", "total_questions": "350"}, {"accuracy": "0.7676191428571428", "provider_name": "Phala", "total_questions": "350"}, {"accuracy": "0.7671692361809045", "provider_name": "PrimeIntellect", "total_questions": "398"}, {"accuracy": "0.7624242727272728", "provider_name": "Reka", "total_questions": "550"}, {"accuracy": "0.7633332500000001", "provider_name": "Relace", "total_questions": "200"}, {"accuracy": "0.77", "provider_name": "Sail Research", "total_questions": "750"}, {"accuracy": "0.7666667142857143", "provider_name": "SiliconFlow", "total_questions": "350"}, {"accuracy": "0.773333", "provider_name": "StreamLake", "total_questions": "50"}, {"accuracy": "0.817778", "provider_name": "Tenstorrent", "total_questions": "150"}, {"accuracy": "0.7838094285714285", "provider_name": "Together", "total_questions": "350"}, {"accuracy": "0.7695238571428571", "provider_name": "Venice", "total_questions": "350"}, {"accuracy": "0.7900492128514055", "provider_name": "Wafer", "total_questions": "747"}, {"accuracy": "0.78083325", "provider_name": "Z.AI", "total_questions": "400"}], "rule": "first-party provider (1 of 38 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.48", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-10-14T16:16:30.000Z", "id": "YGX23vJ7GJBaYUVLvHnXn5", "mean_score": "0.48", "stderr": "0.05021167315686782"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.38333333333333336, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.38333333333333336", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-09-10T20:22:18.000Z", "id": "9te7vbqxpjSaQgHagpaTFg", "mean_score": "0.38333333333333336", "stderr": "0.06329764084940143"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-06", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_high", "source_row": {"Best score (across scorers)": "0.23", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_high", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-08-06T14:07:21.000Z", "id": "AsFSc8AQu4F46Tzp4xVwYu", "mean_score": "0.23", "stderr": "0.04229525846816508"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.317, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.317", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-06-11T14:48:04.000Z", "id": "J3NUxsK2GXq9Whkjn7mEVm", "mean_score": "0.317", "stderr": "0.073"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.674, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.674", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-06-11T02:48:30.000Z", "id": "PeZ84GTQdSYQzDxRMWA2Ha", "mean_score": "0.674", "stderr": "0.028"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-12-11", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9611111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_high", "source_row": {"Best score (across scorers)": "0.9611111111111111", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_high", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2025-12-11T20:49:15.912Z", "id": "LiUMZyNSM8q93KDubWgYCh", "mean_score": "0.9611111111111111", "stderr": "0.025997226390952678"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-12", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.737603305785124, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_high", "source_row": {"Best score (across scorers)": "0.737603305785124", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_high", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-02-12T13:34:00.865Z", "id": "9owdbpnWJ5pkQQC3GWKP5L", "mean_score": "0.737603305785124", "stderr": "0.020017832064928277"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-12-13", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.914, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.914", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2025-12-13T09:04:00.000Z", "id": "Ejo6Fy7XYxUDcCwbejfcYX", "mean_score": "0.914", "stderr": "0.018"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.878788, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.2-20251211", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.2-20251211", "rows": [{"accuracy": "0.878788", "provider_name": "OpenAI", "total_questions": "198"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.371, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.371", "Country": "United States of America", "Model version": "gpt-5.2-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-08-27T19:27:55.000Z", "id": "E3325ytkSNYP8ez53VA4fE", "mean_score": "0.371", "stderr": "0.015283736211823095"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5587, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "GPT-5.2-2025-12-11 (FC)", "source_row": {"Model": "GPT-5.2-2025-12-11 (FC)", "Organization": "OpenAI", "Overall Acc": "55.87%", "Rank": "16"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "gpt-5-2", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.726667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.2-20251211", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.2-20251211", "rows": [{"accuracy": "0.7", "provider_name": "Azure", "total_questions": "50"}, {"accuracy": "0.726667", "provider_name": "OpenAI", "total_questions": "50"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.26", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-10-13T13:33:00.000Z", "id": "cbQnfsMSA3aQaBtjm3aFx2", "mean_score": "0.26", "stderr": "0.04408440022768081"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-28", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.32, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.32", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-07-28T19:23:45.000Z", "id": "f7dotY9NheKNKK2zz6bfKZ", "mean_score": "0.32", "stderr": "0.04688261722621506"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-13", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34146341463414637, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.34146341463414637", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-06-13T03:08:22.000Z", "id": "FHZCKbtfvpKGXU99vZFnXN", "mean_score": "0.34146341463414637", "stderr": "0.07497768853141171"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-13", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6456140350877193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.6456140350877193", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-06-13T03:08:22.000Z", "id": "RG2biwnmGybSVTVxxMdRWa", "mean_score": "0.6456140350877193", "stderr": "0.02838347520543562"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.9555555555555556", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-08-07T00:39:02.000Z", "id": "D3YDUZSfEUDsS4frLbEKDu", "mean_score": "0.9555555555555556", "stderr": "0.031067790907534736"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-06-18", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7727272727272727, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.7727272727272727", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-06-18T17:48:14.587Z", "id": "83hhDhcT9UUrZ2NiUbWSUU", "mean_score": "0.7727272727272727", "stderr": "0.01906835447169652"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9090909090909091, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.9090909090909091", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-08-07T00:46:28.000Z", "id": "W6mTanyjvXhasFBzHukv4S", "mean_score": "0.9090909090909091", "stderr": "0.020482086775424225"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-09-27", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9090909999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.7-max-20260520", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-max-20260520", "rows": [{"accuracy": "0.9090909999999999", "provider_name": "AtlasCloud", "total_questions": "198"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5576730190571715, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-max", "source_row": {"Best score (across scorers)": "0.5576730190571715", "Country": "China", "Model version": "qwen3.7-max", "Organization": "Alibaba", "Release date": "2026-05-19", "Started at": "2026-08-27T19:34:09.000Z", "id": "DMjUrG6eZEqwcQ4SqUoViu", "mean_score": "0.5576730190571715", "stderr": "0.015737359397180177"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-09-27", "model": "qwen3-7-max", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.72, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.7-max-20260520", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-max-20260520", "rows": [{"accuracy": "0.72", "provider_name": "AtlasCloud", "total_questions": "50"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.42, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_low", "source_row": {"Best score (across scorers)": "0.42", "Country": "United States of America", "Model version": "gemini-3.6-flash_low", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-10-13T13:12:52.000Z", "id": "hJ6XXLBm9yfXtSbbhWuwpw", "mean_score": "0.42", "stderr": "0.04960449637488583"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23333333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_high", "source_row": {"Best score (across scorers)": "0.23333333333333334", "Country": "United States of America", "Model version": "gemini-3.6-flash_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-09-10T20:23:53.000Z", "id": "Kkf7BH3sbSqhEDhhPLcmDN", "mean_score": "0.23333333333333334", "stderr": "0.05376590116477416"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-05", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_high", "source_row": {"Best score (across scorers)": "0.3", "Country": "United States of America", "Model version": "gemini-3.6-flash_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-05T01:49:56.000Z", "id": "YBqiCXPjMUSAgAjDUoz5sy", "mean_score": "0.3", "stderr": "0.04605661864718382"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-02", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21951219512195122, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_high", "source_row": {"Best score (across scorers)": "0.21951219512195122", "Country": "United States of America", "Model version": "gemini-3.6-flash_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-02T00:36:45.000Z", "id": "JCMjXU6FRiiLV3xvs9R2Xv", "mean_score": "0.21951219512195122", "stderr": "0.06544589202438408"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-02", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5894736842105263, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_high", "source_row": {"Best score (across scorers)": "0.5894736842105263", "Country": "United States of America", "Model version": "gemini-3.6-flash_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-02T00:36:45.000Z", "id": "nb9LnDeeQyvRCqsbsTANGD", "mean_score": "0.5894736842105263", "stderr": "0.02919063494391404"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-02", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9416666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_high", "source_row": {"Best score (across scorers)": "0.9416666666666667", "Country": "United States of America", "Model version": "gemini-3.6-flash_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-02T00:21:27.000Z", "id": "ajQFZGMjvyMTH2ULDZhwtz", "mean_score": "0.9416666666666667", "stderr": "0.030926354359015452"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-02", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9412878787878788, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_high", "source_row": {"Best score (across scorers)": "0.9412878787878788", "Country": "United States of America", "Model version": "gemini-3.6-flash_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-02T00:21:27.000Z", "id": "86SY5JbPB49xXhHFQW6tHu", "mean_score": "0.9412878787878788", "stderr": "0.014028967942064968"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.930415, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.6-flash-20260721", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.6-flash-20260721", "rows": [{"accuracy": "0.930415", "provider_name": "Google", "total_questions": "1188"}, {"accuracy": "0.9325395714285714", "provider_name": "Google AI Studio", "total_questions": "756"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.662, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.6-flash_high", "source_row": {"Best score (across scorers)": "0.662", "Country": "United States of America", "Model version": "gemini-3.6-flash_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-27T19:30:09.000Z", "id": "F83pa7Gfsb4UNVttTnS5Hk", "mean_score": "0.662", "stderr": "0.014965960710224383"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-3-6-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.72, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.6-flash-20260721", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.6-flash-20260721", "rows": [{"accuracy": "0.7590477142857143", "provider_name": "Google", "total_questions": "350"}, {"accuracy": "0.72", "provider_name": "Google AI Studio", "total_questions": "200"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.32, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-flash-0731_max", "source_row": {"Best score (across scorers)": "0.32", "Country": "China", "Model version": "deepseek-v4-flash-0731_max", "Organization": "DeepSeek", "Release date": "2026-07-31", "Started at": "2026-10-13T13:32:51.000Z", "Training compute (FLOP)": "2.496e+24", "id": "Azz52eFkXwVJiRCGpaUBwB", "mean_score": "0.32", "stderr": "0.04688261722621506"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-05", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-flash-0731_max", "source_row": {"Best score (across scorers)": "0.34", "Country": "China", "Model version": "deepseek-v4-flash-0731_max", "Organization": "DeepSeek", "Release date": "2026-07-31", "Started at": "2026-08-05T23:40:26.000Z", "Training compute (FLOP)": "2.496e+24", "id": "4S4GXbkRhLz8wWP97jYG57", "mean_score": "0.34", "stderr": "0.04760952285695233"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-02", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.24390243902439024, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-flash-0731_max", "source_row": {"Best score (across scorers)": "0.24390243902439024", "Country": "China", "Model version": "deepseek-v4-flash-0731_max", "Organization": "DeepSeek", "Release date": "2026-07-31", "Started at": "2026-08-02T02:15:37.000Z", "Training compute (FLOP)": "2.496e+24", "id": "WYs83xB6NwrkMNyQJtuMcQ", "mean_score": "0.24390243902439024", "stderr": "0.06789956540036612"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-02", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5754385964912281, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-flash-0731_max", "source_row": {"Best score (across scorers)": "0.5754385964912281", "Country": "China", "Model version": "deepseek-v4-flash-0731_max", "Organization": "DeepSeek", "Release date": "2026-07-31", "Started at": "2026-08-02T02:15:37.000Z", "Training compute (FLOP)": "2.496e+24", "id": "mKhVMEicZeGjdvJdTRd7mK", "mean_score": "0.5754385964912281", "stderr": "0.029329899789943818"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-02", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9444444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-flash-0731_max", "source_row": {"Best score (across scorers)": "0.9444444444444444", "Country": "China", "Model version": "deepseek-v4-flash-0731_max", "Organization": "DeepSeek", "Release date": "2026-07-31", "Started at": "2026-08-02T00:22:31.000Z", "Training compute (FLOP)": "2.496e+24", "id": "FQ3Suh2vKacz7UNqJybTWk", "mean_score": "0.9444444444444444", "stderr": "0.020854366486931897"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-02", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9103535353535354, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-flash-0731_max", "source_row": {"Best score (across scorers)": "0.9103535353535354", "Country": "China", "Model version": "deepseek-v4-flash-0731_max", "Organization": "DeepSeek", "Release date": "2026-07-31", "Started at": "2026-08-02T00:22:31.000Z", "Training compute (FLOP)": "2.496e+24", "id": "EDFusDaaFkh5n2MqKNxMBD", "mean_score": "0.9103535353535354", "stderr": "0.017655466909172534"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-08", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8996633999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-flash-20260731", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "rows": [{"accuracy": "0.8765966933797908", "provider_name": "AkashML", "total_questions": "1148"}, {"accuracy": "0.9038162150401354", "provider_name": "Alibaba", "total_questions": "2367"}, {"accuracy": "0.8613915000000001", "provider_name": "Ambient", "total_questions": "1188"}, {"accuracy": "0.8856619999999997", "provider_name": "AtlasCloud", "total_questions": "2376"}, {"accuracy": "0.8995076050632912", "provider_name": "Baidu", "total_questions": "2370"}, {"accuracy": "0.8596491924664603", "provider_name": "BaseTen", "total_questions": "3876"}, {"accuracy": "0.8564731094373864", "provider_name": "Cloudflare", "total_questions": "5510"}, {"accuracy": "0.8949492", "provider_name": "Cohere", "total_questions": "990"}, {"accuracy": "0.8721831538461537", "provider_name": "CoreWeave", "total_questions": "2574"}, {"accuracy": "0.8798644568527919", "provider_name": "Decart", "total_questions": "591"}, {"accuracy": "0.8942210023995201", "provider_name": "DeepInfra", "total_questions": "3334"}, {"accuracy": "0.8996633999999999", "provider_name": "DeepSeek", "total_questions": "990"}, {"accuracy": "0.8561369999999999", "provider_name": "DigitalOcean", "total_questions": "2178"}, {"accuracy": "0.8787801424227066", "provider_name": "Fireworks", "total_questions": "1973"}, {"accuracy": "0.8921505000000001", "provider_name": "GMICloud", "total_questions": "3168"}, {"accuracy": "0.8808310763358778", "provider_name": "Inceptron", "total_questions": "2358"}, {"accuracy": "0.8312629347826086", "provider_name": "Io Net", "total_questions": "644"}, {"accuracy": "0.8699495", "provider_name": "Ionstream", "total_questions": "396"}, {"accuracy": "0.8745317312013828", "provider_name": "Makora", "total_questions": "1157"}, {"accuracy": "0.8713910913385827", "provider_name": "Mancer 2", "total_questions": "2540"}, {"accuracy": "0.8612792", "provider_name": "Morph", "total_questions": "990"}, {"accuracy": "0.836139", "provider_name": "Nebius", "total_questions": "594"}, {"accuracy": "0.8993638888888887", "provider_name": "NextBit", "total_questions": "1782"}, {"accuracy": "0.896873431174089", "provider_name": "Novita", "total_questions": "2964"}, {"accuracy": "0.7547698333333334", "provider_name": "OpenInference", "total_questions": "1188"}, {"accuracy": "0.8856209104124406", "provider_name": "Parasail", "total_questions": "2958"}, {"accuracy": "0.8855217500000001", "provider_name": "Phala", "total_questions": "2376"}, {"accuracy": "0.8890626292530308", "provider_name": "Reka", "total_questions": "2557"}, {"accuracy": "0.8867924431103489", "provider_name": "Relace", "total_questions": "1749"}, {"accuracy": "0.8444203561384532", "provider_name": "Sail Research", "total_questions": "3698"}, {"accuracy": "0.8951084117647057", "provider_name": "SiliconFlow", "total_questions": "3162"}, {"accuracy": "0.8821546153846153", "provider_name": "StreamLake", "total_questions": "2574"}, {"accuracy": "0.8721539397539244", "provider_name": "Together", "total_questions": "2357"}, {"accuracy": "0.8733865833333335", "provider_name": "Venice", "total_questions": "2376"}, {"accuracy": "0.8833942549450549", "provider_name": "Wafer", "total_questions": "2730"}], "rule": "first-party provider (1 of 35 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.33633633633633636, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-flash-0731_max", "source_row": {"Best score (across scorers)": "0.33633633633633636", "Country": "China", "Model version": "deepseek-v4-flash-0731_max", "Organization": "DeepSeek", "Release date": "2026-07-31", "Started at": "2026-08-27T19:30:16.000Z", "Training compute (FLOP)": "2.496e+24", "id": "TqFmiGvqozt7TSZYmhgF8m", "mean_score": "0.33633633633633636", "stderr": "0.01495531772489948"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-08", "model": "deepseek-v4-flash-0731", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7766665, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-flash-20260731", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260731", "rows": [{"accuracy": "0.7426668", "provider_name": "AkashML", "total_questions": "250"}, {"accuracy": "0.7582582837837838", "provider_name": "Alibaba", "total_questions": "444"}, {"accuracy": "0.7439998000000001", "provider_name": "Ambient", "total_questions": "250"}, {"accuracy": "0.7479999", "provider_name": "AtlasCloud", "total_questions": "500"}, {"accuracy": "0.7526667", "provider_name": "Baidu", "total_questions": "500"}, {"accuracy": "0.8368423578947368", "provider_name": "BaseTen", "total_questions": "95"}, {"accuracy": "0.7561904285714286", "provider_name": "Cloudflare", "total_questions": "700"}, {"accuracy": "0.7670682771084337", "provider_name": "Cohere", "total_questions": "249"}, {"accuracy": "0.7583333333333333", "provider_name": "CoreWeave", "total_questions": "600"}, {"accuracy": "0.7333333333333333", "provider_name": "Decart", "total_questions": "150"}, {"accuracy": "0.7537594536340853", "provider_name": "DeepInfra", "total_questions": "798"}, {"accuracy": "0.7766665", "provider_name": "DeepSeek", "total_questions": "300"}, {"accuracy": "0.6465773571428571", "provider_name": "DigitalOcean", "total_questions": "448"}, {"accuracy": "0.7505555", "provider_name": "Fireworks", "total_questions": "300"}, {"accuracy": "0.7688889333333333", "provider_name": "GMICloud", "total_questions": "750"}, {"accuracy": "0.7440801639344262", "provider_name": "Inceptron", "total_questions": "549"}, {"accuracy": "0.7448276551724138", "provider_name": "Io Net", "total_questions": "145"}, {"accuracy": "0.7542953608247424", "provider_name": "Ionstream", "total_questions": "97"}, {"accuracy": "0.7210882448979592", "provider_name": "Makora", "total_questions": "245"}, {"accuracy": "0.7362962222222222", "provider_name": "Mancer 2", "total_questions": "450"}, {"accuracy": "0.7406779661016949", "provider_name": "Morph", "total_questions": "295"}, {"accuracy": "0.675675972972973", "provider_name": "Nebius", "total_questions": "148"}, {"accuracy": "0.76666675", "provider_name": "NextBit", "total_questions": "400"}, {"accuracy": "0.7576190714285714", "provider_name": "Novita", "total_questions": "700"}, {"accuracy": "0.7325", "provider_name": "OpenInference", "total_questions": "200"}, {"accuracy": "0.7600536219839142", "provider_name": "Parasail", "total_questions": "746"}, {"accuracy": "0.7496969090909091", "provider_name": "Phala", "total_questions": "550"}, {"accuracy": "0.7472220833333334", "provider_name": "Reka", "total_questions": "600"}, {"accuracy": "0.7466666000000001", "provider_name": "Relace", "total_questions": "500"}, {"accuracy": "0.7649999375000001", "provider_name": "Sail Research", "total_questions": "800"}, {"accuracy": "0.7697776666666667", "provider_name": "SiliconFlow", "total_questions": "750"}, {"accuracy": "0.7611110833333333", "provider_name": "StreamLake", "total_questions": "600"}, {"accuracy": "0.7629022732240437", "provider_name": "Together", "total_questions": "549"}, {"accuracy": "0.7575758181818182", "provider_name": "Venice", "total_questions": "550"}, {"accuracy": "0.7566666666666667", "provider_name": "Wafer", "total_questions": "600"}], "rule": "first-party provider (1 of 35 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_max", "source_row": {"Best score (across scorers)": "0.21", "Country": "United States of America", "Model version": "claude-opus-4-6_max", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-10-13T13:43:19.000Z", "id": "aKc2rzsA6YPYwCkE7nhvvB", "mean_score": "0.21", "stderr": "0.040936018074033236"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2833333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_max", "source_row": {"Best score (across scorers)": "0.2833333333333333", "Country": "United States of America", "Model version": "claude-opus-4-6_max", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-09-10T20:22:53.000Z", "id": "4Jq9M5CbuWMyxRHHxavMwA", "mean_score": "0.2833333333333333", "stderr": "0.05866531842313124"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-25", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.25, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_max", "source_row": {"Best score (across scorers)": "0.25", "Country": "United States of America", "Model version": "claude-opus-4-6_max", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-07-25T01:44:09.000Z", "id": "64rokFyL2L8Vd4SZVdayCy", "mean_score": "0.25", "stderr": "0.04351941398892446"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2682926829268293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_max", "source_row": {"Best score (across scorers)": "0.2682926829268293", "Country": "United States of America", "Model version": "claude-opus-4-6_max", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-06-11T00:19:44.000Z", "id": "2hbbp8z6PhB7c2T33os9q5", "mean_score": "0.2682926829268293", "stderr": "0.07005564203095158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6596491228070176, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_max", "source_row": {"Best score (across scorers)": "0.6596491228070176", "Country": "United States of America", "Model version": "claude-opus-4-6_max", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-06-11T00:19:44.000Z", "id": "kgJ2d6XbKj3dZutDcBYSKU", "mean_score": "0.6596491228070176", "stderr": "0.028116467881852295"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-02-06", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9444444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_64K", "source_row": {"Best score (across scorers)": "0.9444444444444444", "Country": "United States of America", "Model version": "claude-opus-4-6_64K", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-02-06T04:32:30.354Z", "id": "TUPC8VdX4jApFa4MGfNP4t", "mean_score": "0.9444444444444444", "stderr": "0.028247294597471075"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-18", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7871900826446281, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6", "source_row": {"Best score (across scorers)": "0.7871900826446281", "Country": "United States of America", "Model version": "claude-opus-4-6", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-02-18T20:07:57.705Z", "id": "2hH2GAPUFaJR7pfVKhhicg", "mean_score": "0.7871900826446281", "stderr": "0.01862353730305288"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-02-06", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9053030303030303, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_32K", "source_row": {"Best score (across scorers)": "0.9053030303030303", "Country": "United States of America", "Model version": "claude-opus-4-6_32K", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-02-06T02:17:07.741Z", "id": "DiXCMhdfhcjaCUSzYWgACy", "mean_score": "0.9053030303030303", "stderr": "0.016916245311450143"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.856643153846154, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.6-opus-20260205", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "rows": [{"accuracy": "0.8624119090909091", "provider_name": "Amazon Bedrock", "total_questions": "2178"}, {"accuracy": "0.856643153846154", "provider_name": "Anthropic", "total_questions": "2574"}, {"accuracy": "0.8585856363636363", "provider_name": "Azure", "total_questions": "2178"}, {"accuracy": "0.8491582000000001", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.8561369999999997", "provider_name": "Google", "total_questions": "4356"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.47, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-6_max", "source_row": {"Best score (across scorers)": "0.47", "Country": "United States of America", "Model version": "claude-opus-4-6_max", "Organization": "Anthropic", "Release date": "2026-02-05", "Started at": "2026-08-27T19:27:45.000Z", "id": "85fhMxLaebWtLoi6oLpTbi", "mean_score": "0.47", "stderr": "0.015790799515836722"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-opus-4-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7638095, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.6-opus-20260205", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-opus-20260205", "rows": [{"accuracy": "0.7521212727272727", "provider_name": "Amazon Bedrock", "total_questions": "550"}, {"accuracy": "0.7638095", "provider_name": "Anthropic", "total_questions": "700"}, {"accuracy": "0.7484848181818182", "provider_name": "Azure", "total_questions": "550"}, {"accuracy": "0.7414814444444444", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7512120909090909", "provider_name": "Google", "total_questions": "1100"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.39, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview_high", "source_row": {"Best score (across scorers)": "0.39", "Country": "United States of America", "Model version": "gemini-3-flash-preview_high", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-10-13T13:42:10.000Z", "id": "UUNZJtgib9XbCjLEdhtGaT", "mean_score": "0.39", "stderr": "0.04902071300001973"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-05", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview_low", "source_row": {"Best score (across scorers)": "0.26", "Country": "United States of America", "Model version": "gemini-3-flash-preview_low", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-08-05T23:47:24.000Z", "id": "ZQMcTQjHcDSevCN2dYYHKY", "mean_score": "0.26", "stderr": "0.0440844002276808"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17073170731707318, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview", "source_row": {"Best score (across scorers)": "0.17073170731707318", "Country": "United States of America", "Model version": "gemini-3-flash-preview", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-06-11T23:44:30.000Z", "id": "ZczEhu69FQ9Cnns8NT6ew4", "mean_score": "0.17073170731707318", "stderr": "0.05949419959829497"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.512280701754386, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview", "source_row": {"Best score (across scorers)": "0.512280701754386", "Country": "United States of America", "Model version": "gemini-3-flash-preview", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-06-11T23:44:30.000Z", "id": "EtgrDq6ei6kZe8K7QbXUe3", "mean_score": "0.512280701754386", "stderr": "0.02966059084324674"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-06", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview_high", "source_row": {"Best score (across scorers)": "0.9555555555555556", "Country": "United States of America", "Model version": "gemini-3-flash-preview_high", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-08-06T22:25:57.000Z", "id": "a4uE6KgMhGSUjcnemRnbkA", "mean_score": "0.9555555555555556", "stderr": "0.031067790907534743"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-18", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7541322314049587, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview", "source_row": {"Best score (across scorers)": "0.7541322314049587", "Country": "United States of America", "Model version": "gemini-3-flash-preview", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-02-18T20:12:36.725Z", "id": "jxoyMNmTnYirjLJN6wRNmG", "mean_score": "0.7541322314049587", "stderr": "0.019593002139508684"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8939393939393939, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview_high", "source_row": {"Best score (across scorers)": "0.8939393939393939", "Country": "United States of America", "Model version": "gemini-3-flash-preview_high", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-08-06T22:22:58.000Z", "id": "gVdraDT3pXmaExP9VbqME4", "mean_score": "0.8939393939393939", "stderr": "0.021938047738853106"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.8855220000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3-flash-preview-20251217", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3-flash-preview-20251217", "rows": [{"accuracy": "0.8905723333333333", "provider_name": "Google", "total_questions": "594"}, {"accuracy": "0.8855220000000001", "provider_name": "Google AI Studio", "total_questions": "594"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.668, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-flash-preview_high", "source_row": {"Best score (across scorers)": "0.668", "Country": "United States of America", "Model version": "gemini-3-flash-preview_high", "Organization": "Google DeepMind", "Release date": "2025-12-17", "Started at": "2026-08-27T19:30:06.000Z", "id": "7cc6cDCXjsPjtKt36bbeMR", "mean_score": "0.668", "stderr": "0.014899597242811567"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-3-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7533333333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3-flash-preview-20251217", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3-flash-preview-20251217", "rows": [{"accuracy": "0.7533333333333333", "provider_name": "Google", "total_questions": "150"}, {"accuracy": "0.7577776666666667", "provider_name": "Google AI Studio", "total_questions": "150"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.2", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-10-13T17:16:43.000Z", "id": "ehqiuEDETZTqJ8zt6Mc7Ja", "mean_score": "0.2", "stderr": "0.04020151261036849"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-11", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.21666666666666667", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-09-11T02:02:56.000Z", "id": "eJcV4jnwhxD6tTxNkrn9Xn", "mean_score": "0.21666666666666667", "stderr": "0.052301098424620544"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-17", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.18, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.18", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-07-17T23:54:57.000Z", "id": "RA5MRbmWJpFPR29BQxfvZt", "mean_score": "0.18", "stderr": "0.03861229196653691"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-10", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2564102564102564, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.2564102564102564", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-06-10T19:47:36.000Z", "id": "eVKA2eavdRAbSN6AnzyBxA", "mean_score": "0.2564102564102564", "stderr": "0.07083413480167725"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-10", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5719298245614035, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.5719298245614035", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-06-10T19:47:36.000Z", "id": "BxrNfyMffkVUq4n7U5eK3Q", "mean_score": "0.5719298245614035", "stderr": "0.029360921879029937"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-05-02", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9611111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.9611111111111111", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-05-02T09:08:49.000Z", "id": "BTeRdhgt6TFBj6QkGX2j6P", "mean_score": "0.9611111111111111", "stderr": "0.023777636174410793"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-05-08", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7665289256198347, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.7665289256198347", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-05-08T10:21:24.110Z", "id": "bUPtAK7spEwvmiC8pi6afc", "mean_score": "0.7665289256198347", "stderr": "0.01924895909142592"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-05-01", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9078282828282829, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.9078282828282829", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-05-01T17:25:36.000Z", "id": "LjgJoDosTLkiGVT59smA7q", "mean_score": "0.9078282828282829", "stderr": "0.0171745162866666"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8799901428571427, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2.6-20260420", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "rows": [{"accuracy": "0.7919672857142857", "provider_name": "AtlasCloud", "total_questions": "1386"}, {"accuracy": "0.8718132857142857", "provider_name": "Baidu", "total_questions": "1386"}, {"accuracy": "0.8571427142857143", "provider_name": "Chutes", "total_questions": "1386"}, {"accuracy": "0.8741024915384615", "provider_name": "Cloudflare", "total_questions": "1300"}, {"accuracy": "0.86847625", "provider_name": "CoreWeave", "total_questions": "1584"}, {"accuracy": "0.8478604104595879", "provider_name": "Crusoe", "total_questions": "1262"}, {"accuracy": "0.8749014196217493", "provider_name": "Decart", "total_questions": "846"}, {"accuracy": "0.6200336919191919", "provider_name": "DeepInfra", "total_questions": "990"}, {"accuracy": "0.7989417142857143", "provider_name": "DigitalOcean", "total_questions": "1386"}, {"accuracy": "0.8830752947678703", "provider_name": "Inceptron", "total_questions": "1357"}, {"accuracy": "0.8799901428571427", "provider_name": "Moonshot AI", "total_questions": "1386"}, {"accuracy": "0.8790281428571429", "provider_name": "Novita", "total_questions": "1386"}, {"accuracy": "0.8759949037313433", "provider_name": "Parasail", "total_questions": "1340"}, {"accuracy": "0.8558689848156183", "provider_name": "Phala", "total_questions": "1383"}, {"accuracy": "0.8871941263616558", "provider_name": "SiliconFlow", "total_questions": "1377"}, {"accuracy": "0.845748875", "provider_name": "StreamLake", "total_questions": "1584"}, {"accuracy": "0.8549618244274809", "provider_name": "Together", "total_questions": "393"}, {"accuracy": "0.6560053838120105", "provider_name": "Venice", "total_questions": "766"}], "rule": "first-party provider (1 of 18 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.349, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.6", "source_row": {"Best score (across scorers)": "0.349", "Country": "China", "Model version": "kimi-k2.6", "Organization": "Moonshot", "Release date": "2026-04-20", "Started at": "2026-08-10T21:35:30.000Z", "id": "hHnQVatVTbQjM4sfrjw7GH", "mean_score": "0.349", "stderr": "0.015080663991563052"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "kimi-k2-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7671130223214285, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2.6-20260420", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.6-20260420", "rows": [{"accuracy": "0.7088888888888889", "provider_name": "AtlasCloud", "total_questions": "450"}, {"accuracy": "0.738333375", "provider_name": "Baidu", "total_questions": "400"}, {"accuracy": "0.728333125", "provider_name": "Chutes", "total_questions": "400"}, {"accuracy": "0.7500001", "provider_name": "Cloudflare", "total_questions": "500"}, {"accuracy": "0.7585186666666667", "provider_name": "CoreWeave", "total_questions": "450"}, {"accuracy": "0.7418154508928572", "provider_name": "Crusoe", "total_questions": "448"}, {"accuracy": "0.7167631098265895", "provider_name": "Decart", "total_questions": "346"}, {"accuracy": "0.7464314105793451", "provider_name": "DeepInfra", "total_questions": "397"}, {"accuracy": "0.740963843373494", "provider_name": "DigitalOcean", "total_questions": "498"}, {"accuracy": "0.713333", "provider_name": "GMICloud", "total_questions": "50"}, {"accuracy": "0.7338531336302895", "provider_name": "Inceptron", "total_questions": "449"}, {"accuracy": "0.7671130223214285", "provider_name": "Moonshot AI", "total_questions": "448"}, {"accuracy": "0.75583325", "provider_name": "Novita", "total_questions": "400"}, {"accuracy": "0.7357142857142858", "provider_name": "Parasail", "total_questions": "350"}, {"accuracy": "0.745833375", "provider_name": "Phala", "total_questions": "400"}, {"accuracy": "0.7466667499999999", "provider_name": "SiliconFlow", "total_questions": "400"}, {"accuracy": "0.7473841614349777", "provider_name": "StreamLake", "total_questions": "446"}, {"accuracy": "0.7400001428571428", "provider_name": "Venice", "total_questions": "350"}], "rule": "first-party provider (1 of 18 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.36, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.36", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-10-13T14:07:43.000Z", "Training compute (FLOP)": "6.6e+25", "id": "CxCwksDLsyrgTJ5ievyAoq", "mean_score": "0.36", "stderr": "0.04824181513244218"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-05", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.23", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-08-05T19:32:49.000Z", "Training compute (FLOP)": "6.6e+25", "id": "NKdPCHXg9sphwKW2UEnmkD", "mean_score": "0.23", "stderr": "0.04229525846816508"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21951219512195122, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.21951219512195122", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-06-11T01:44:04.000Z", "Training compute (FLOP)": "6.6e+25", "id": "mSS7C84hJyYi4VnJtgzYvW", "mean_score": "0.21951219512195122", "stderr": "0.06544589202438408"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-10", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5543859649122806, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.5543859649122806", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-06-10T20:10:57.000Z", "Training compute (FLOP)": "6.6e+25", "id": "QRDHTGX3rYrVCjVPRarjjt", "mean_score": "0.5543859649122806", "stderr": "0.029493504108183504"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-10-29", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9813066465256798, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.9813066465256798", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-29T22:29:43.092Z", "Training compute (FLOP)": "6.6e+25", "id": "4wN6FKe2db9CLQ3NUFfYge", "mean_score": "0.9813066465256798", "stderr": "0.0031922231523012308"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-29", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9138888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.9138888888888889", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-29T22:29:41.753Z", "Training compute (FLOP)": "6.6e+25", "id": "3D8aaFBLZwQcnWHYKwvnmq", "mean_score": "0.9138888888888889", "stderr": "0.03658872186755414"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-08-23", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.88, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-5 (high)", "source_row": {"command": "aider --model openai/gpt-5", "date": "2025-08-23", "dirname": "2025-08-23-15-47-21--gpt-5-high", "model": "gpt-5 (high)", "pass_rate_2": 88.0, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-06", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7355371900826446, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.7355371900826446", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-02-06T14:23:51.990Z", "Training compute (FLOP)": "6.6e+25", "id": "C85Bnci9xVzq5jmt8zNnbw", "mean_score": "0.7355371900826446", "stderr": "0.02006832181710289"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-10-29", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8617424242424242, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.8617424242424242", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-29T22:29:36.692Z", "Training compute (FLOP)": "6.6e+25", "id": "9nPyNrZxwtoT7eS4DZbQF6", "mean_score": "0.8617424242424242", "stderr": "0.020787836099851995"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8501685000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5-2025-08-07", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-2025-08-07", "rows": [{"accuracy": "0.8518517499999999", "provider_name": "Azure", "total_questions": "792"}, {"accuracy": "0.8501685000000001", "provider_name": "OpenAI", "total_questions": "396"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.501, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.501", "Country": "United States of America", "Model version": "gpt-5-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-08-27T19:28:49.000Z", "Training compute (FLOP)": "6.6e+25", "id": "gTLurbyKU63i2opsGkJGwh", "mean_score": "0.501", "stderr": "0.015819268290576817"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7066666666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5-2025-08-07", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-2025-08-07", "rows": [{"accuracy": "0.7056452258064517", "provider_name": "Azure", "total_questions": "248"}, {"accuracy": "0.7066666666666667", "provider_name": "OpenAI", "total_questions": "150"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_max", "source_row": {"Best score (across scorers)": "0.22", "Country": "China", "Model version": "glm-5.2_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-10-14T05:10:27.000Z", "id": "Hui8iZgjwKPxz7DkRU8sJX", "mean_score": "0.22", "stderr": "0.041633319989322654"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.19, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_low", "source_row": {"Best score (across scorers)": "0.19", "Country": "China", "Model version": "glm-5.2_low", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-08-27T18:13:46.000Z", "id": "7DQNxGG5q6i7VtnzF7vNxA", "mean_score": "0.19", "stderr": "0.039427724440366255"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-19", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2926829268292683, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_max", "source_row": {"Best score (across scorers)": "0.2926829268292683", "Country": "China", "Model version": "glm-5.2_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-06-19T16:13:05.000Z", "id": "JebA8dYLVov6H49ZQSLXh5", "mean_score": "0.2926829268292683", "stderr": "0.07194088392074452"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-19", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.592057761732852, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_max", "source_row": {"Best score (across scorers)": "0.592057761732852", "Country": "China", "Model version": "glm-5.2_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-06-19T16:13:05.000Z", "id": "TTuCPacMDpNaCSs3Hpbdjp", "mean_score": "0.592057761732852", "stderr": "0.029581952519606228"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-06-25", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8638888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_max", "source_row": {"Best score (across scorers)": "0.8638888888888889", "Country": "China", "Model version": "glm-5.2_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-06-25T00:15:08.000Z", "id": "XjFqou6MyT2XLpkUsCrgxM", "mean_score": "0.8638888888888889", "stderr": "0.04219779330510574"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-06-25", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.787, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_max", "source_row": {"Best score (across scorers)": "0.787", "Country": "China", "Model version": "glm-5.2_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-06-25T13:13:06.902Z", "id": "4AfhcmYVNrw6gM5u8CQsyy", "mean_score": "0.787", "stderr": "0.018656911901656907"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-06-24", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9185606060606061, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_max", "source_row": {"Best score (across scorers)": "0.9185606060606061", "Country": "China", "Model version": "glm-5.2_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-06-24T20:55:39.000Z", "id": "c5kpB4axzmygRywWqBKFmv", "mean_score": "0.9185606060606061", "stderr": "0.016123207533687733"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8334862727272728, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.2-20260616", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.2-20260616", "rows": [{"accuracy": "0.8497263442622951", "provider_name": "AkashML", "total_questions": "366"}, {"accuracy": "0.8433546514886163", "provider_name": "Alibaba", "total_questions": "5139"}, {"accuracy": "0.8507791959910913", "provider_name": "Ambient", "total_questions": "1347"}, {"accuracy": "0.8273644545454546", "provider_name": "AtlasCloud", "total_questions": "2178"}, {"accuracy": "0.853815888888889", "provider_name": "Baidu", "total_questions": "3564"}, {"accuracy": "0.8576645647133758", "provider_name": "BaseTen", "total_questions": "7850"}, {"accuracy": "0.805556", "provider_name": "Chutes", "total_questions": "192"}, {"accuracy": "0.8328165280620154", "provider_name": "Cloudflare", "total_questions": "3225"}, {"accuracy": "0.8271603333333334", "provider_name": "CoreWeave", "total_questions": "1782"}, {"accuracy": "0.8264543613323679", "provider_name": "Crusoe", "total_questions": "1381"}, {"accuracy": "0.8476008905764583", "provider_name": "Decart", "total_questions": "2897"}, {"accuracy": "0.8785058930232558", "provider_name": "DeepInfra", "total_questions": "2365"}, {"accuracy": "0.8733164166666666", "provider_name": "DigitalOcean", "total_questions": "2376"}, {"accuracy": "0.8586479490968801", "provider_name": "Fireworks", "total_questions": "3654"}, {"accuracy": "0.8282827272727273", "provider_name": "Friendli", "total_questions": "2178"}, {"accuracy": "0.7936945454545455", "provider_name": "GMICloud", "total_questions": "2178"}, {"accuracy": "0.8615345381097562", "provider_name": "Inceptron", "total_questions": "1968"}, {"accuracy": "0.8378226666666666", "provider_name": "InferenceNet", "total_questions": "594"}, {"accuracy": "0.851852", "provider_name": "Ionstream", "total_questions": "198"}, {"accuracy": "0.8213506405228759", "provider_name": "Makora", "total_questions": "459"}, {"accuracy": "0.8364074347826086", "provider_name": "Mistral", "total_questions": "4554"}, {"accuracy": "0.7219408075949366", "provider_name": "ModelRun", "total_questions": "395"}, {"accuracy": "0.7576013994334279", "provider_name": "Morph", "total_questions": "1765"}, {"accuracy": "0.8406283333333334", "provider_name": "Nebius", "total_questions": "594"}, {"accuracy": "0.8397788936464089", "provider_name": "Novita", "total_questions": "2172"}, {"accuracy": "0.8307303478466634", "provider_name": "Parasail", "total_questions": "2113"}, {"accuracy": "0.7786195454545455", "provider_name": "Phala", "total_questions": "2178"}, {"accuracy": "0.8772070529695024", "provider_name": "Reka", "total_questions": "623"}, {"accuracy": "0.8091166153846153", "provider_name": "Relace", "total_questions": "585"}, {"accuracy": "0.8522881577868852", "provider_name": "Sail Research", "total_questions": "976"}, {"accuracy": "0.8445056363636364", "provider_name": "SiliconFlow", "total_questions": "2178"}, {"accuracy": "0.8320707500000001", "provider_name": "StreamLake", "total_questions": "2376"}, {"accuracy": "0.8383907751724139", "provider_name": "Together", "total_questions": "2175"}, {"accuracy": "0.8741963636363635", "provider_name": "Venice", "total_questions": "2178"}, {"accuracy": "0.9062234806560137", "provider_name": "Wafer", "total_questions": "2378"}, {"accuracy": "0.8334862727272728", "provider_name": "Z.AI", "total_questions": "2178"}], "rule": "first-party provider (1 of 36 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.342, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.2_max", "source_row": {"Best score (across scorers)": "0.342", "Country": "China", "Model version": "glm-5.2_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-06-16", "Started at": "2026-08-27T19:36:05.000Z", "id": "WgjFZzojpiv54XqUhvfuUG", "mean_score": "0.342", "stderr": "0.015008706182121802"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "glm-5-2", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7169697272727272, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.2-20260616", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.2-20260616", "rows": [{"accuracy": "0.7538460512820513", "provider_name": "AkashML", "total_questions": "195"}, {"accuracy": "0.723680833976834", "provider_name": "Alibaba", "total_questions": "1295"}, {"accuracy": "0.7276191428571428", "provider_name": "Ambient", "total_questions": "350"}, {"accuracy": "0.7361110833333333", "provider_name": "AtlasCloud", "total_questions": "600"}, {"accuracy": "0.723137294117647", "provider_name": "Baidu", "total_questions": "850"}, {"accuracy": "0.7075098814229249", "provider_name": "BaseTen", "total_questions": "1518"}, {"accuracy": "0.7619049795918367", "provider_name": "Chutes", "total_questions": "98"}, {"accuracy": "0.7060416875", "provider_name": "Cloudflare", "total_questions": "800"}, {"accuracy": "0.7192592222222223", "provider_name": "CoreWeave", "total_questions": "450"}, {"accuracy": "0.7300001666666667", "provider_name": "Crusoe", "total_questions": "300"}, {"accuracy": "0.7059999333333333", "provider_name": "Decart", "total_questions": "750"}, {"accuracy": "0.7161111666666666", "provider_name": "DeepInfra", "total_questions": "600"}, {"accuracy": "0.7394871538461538", "provider_name": "DigitalOcean", "total_questions": "650"}, {"accuracy": "0.7344348103448277", "provider_name": "Fireworks", "total_questions": "696"}, {"accuracy": "0.688485", "provider_name": "Friendli", "total_questions": "550"}, {"accuracy": "0.6975756363636364", "provider_name": "GMICloud", "total_questions": "550"}, {"accuracy": "0.7061769766277128", "provider_name": "Inceptron", "total_questions": "599"}, {"accuracy": "0.7333333333333333", "provider_name": "InferenceNet", "total_questions": "150"}, {"accuracy": "0.753333", "provider_name": "Ionstream", "total_questions": "50"}, {"accuracy": "0.76", "provider_name": "Makora", "total_questions": "100"}, {"accuracy": "0.6959998500000001", "provider_name": "Mistral", "total_questions": "1000"}, {"accuracy": "0.706667", "provider_name": "ModelRun", "total_questions": "50"}, {"accuracy": "0.7633818649635037", "provider_name": "Morph", "total_questions": "548"}, {"accuracy": "0.7188886666666667", "provider_name": "Nebius", "total_questions": "150"}, {"accuracy": "0.7250608065693431", "provider_name": "Novita", "total_questions": "548"}, {"accuracy": "0.709697", "provider_name": "Parasail", "total_questions": "550"}, {"accuracy": "0.7173648633879781", "provider_name": "Phala", "total_questions": "549"}, {"accuracy": "0.6958335", "provider_name": "Reka", "total_questions": "200"}, {"accuracy": "0.7155553333333334", "provider_name": "Relace", "total_questions": "150"}, {"accuracy": "0.7466665", "provider_name": "Sail Research", "total_questions": "300"}, {"accuracy": "0.7299999166666666", "provider_name": "SiliconFlow", "total_questions": "600"}, {"accuracy": "0.7166665833333333", "provider_name": "StreamLake", "total_questions": "600"}, {"accuracy": "0.7175757272727272", "provider_name": "Together", "total_questions": "550"}, {"accuracy": "0.72722225", "provider_name": "Venice", "total_questions": "600"}, {"accuracy": "0.7213049198113207", "provider_name": "Wafer", "total_questions": "848"}, {"accuracy": "0.7169697272727272", "provider_name": "Z.AI", "total_questions": "550"}], "rule": "first-party provider (1 of 36 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3-flash_max", "source_row": {"Best score (across scorers)": "0.21", "Country": "China", "Model version": "glm-5.3-flash_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-20", "Started at": "2026-10-13T15:54:34.000Z", "Training compute (FLOP)": "3.24e+24", "id": "fWNxmiSUC7BX8whqvzssY8", "mean_score": "0.21", "stderr": "0.040936018074033236"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-28", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3-flash_max", "source_row": {"Best score (across scorers)": "0.08", "Country": "China", "Model version": "glm-5.3-flash_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-20", "Started at": "2026-08-28T14:38:27.000Z", "Training compute (FLOP)": "3.24e+24", "id": "TMM3rPrAHcpBG5Pvw9kr4L", "mean_score": "0.08", "stderr": "0.027265992434429072"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-27", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17073170731707318, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3-flash_max", "source_row": {"Best score (across scorers)": "0.17073170731707318", "Country": "China", "Model version": "glm-5.3-flash_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-20", "Started at": "2026-08-27T13:29:52.000Z", "Training compute (FLOP)": "3.24e+24", "id": "iCysWvFJevNyw2dbYosU6J", "mean_score": "0.17073170731707318", "stderr": "0.05949419959829495"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5578947368421052, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3-flash_max", "source_row": {"Best score (across scorers)": "0.5578947368421052", "Country": "China", "Model version": "glm-5.3-flash_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-20", "Started at": "2026-08-27T13:29:52.000Z", "Training compute (FLOP)": "3.24e+24", "id": "YC2u9wykbYQg4gdkf76j26", "mean_score": "0.5578947368421052", "stderr": "0.029469977528223976"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-26", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9388888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3-flash_max", "source_row": {"Best score (across scorers)": "0.9388888888888889", "Country": "China", "Model version": "glm-5.3-flash_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-20", "Started at": "2026-08-26T22:26:55.000Z", "Training compute (FLOP)": "3.24e+24", "id": "NCikvVWc5SEyMEbCqvVNb2", "mean_score": "0.9388888888888889", "stderr": "0.021622267380853318"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-26", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9015151515151515, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.3-flash_max", "source_row": {"Best score (across scorers)": "0.9015151515151515", "Country": "China", "Model version": "glm-5.3-flash_max", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-08-20", "Started at": "2026-08-26T22:26:55.000Z", "Training compute (FLOP)": "3.24e+24", "id": "Jvt5BNr4ERbE9QGgMcEbmT", "mean_score": "0.9015151515151515", "stderr": "0.017183231147411448"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9090909999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.3-flash", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.3-flash", "rows": [{"accuracy": "0.9090909999999999", "provider_name": "digitalocean", "total_questions": "198"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "glm-5-3-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.762499875, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.3-flash-20260826", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.3-flash-20260826", "rows": [{"accuracy": "0.7573333999999999", "provider_name": "AtlasCloud", "total_questions": "250"}, {"accuracy": "0.7505556666666666", "provider_name": "BaseTen", "total_questions": "300"}, {"accuracy": "0.7433333333333333", "provider_name": "Cloudflare", "total_questions": "300"}, {"accuracy": "0.752", "provider_name": "CoreWeave", "total_questions": "250"}, {"accuracy": "0.7516665", "provider_name": "Crusoe", "total_questions": "200"}, {"accuracy": "0.7386668", "provider_name": "Decart", "total_questions": "250"}, {"accuracy": "0.759408620967742", "provider_name": "DeepInfra", "total_questions": "248"}, {"accuracy": "0.7666665", "provider_name": "DekaLLM", "total_questions": "100"}, {"accuracy": "0.7255891616161616", "provider_name": "DigitalOcean", "total_questions": "396"}, {"accuracy": "0.7564362073170733", "provider_name": "Fireworks", "total_questions": "492"}, {"accuracy": "0.7403200757575757", "provider_name": "Friendli", "total_questions": "396"}, {"accuracy": "0.746666625", "provider_name": "GMICloud", "total_questions": "400"}, {"accuracy": "0.7510040843373493", "provider_name": "Inceptron", "total_questions": "249"}, {"accuracy": "0.753333375", "provider_name": "InferenceNet", "total_questions": "400"}, {"accuracy": "0.743738", "provider_name": "Io Net", "total_questions": "346"}, {"accuracy": "0.7482990408163266", "provider_name": "Makora", "total_questions": "98"}, {"accuracy": "0.7574072222222222", "provider_name": "Modal", "total_questions": "450"}, {"accuracy": "0.7399998571428572", "provider_name": "Morph", "total_questions": "350"}, {"accuracy": "0.7533333333333333", "provider_name": "Near AI", "total_questions": "150"}, {"accuracy": "0.7537880151515151", "provider_name": "NextBit", "total_questions": "396"}, {"accuracy": "0.771666625", "provider_name": "Novita", "total_questions": "400"}, {"accuracy": "0.7224448316633266", "provider_name": "OpenInference", "total_questions": "499"}, {"accuracy": "0.7573333", "provider_name": "Parasail", "total_questions": "500"}, {"accuracy": "0.7533331666666667", "provider_name": "Phala", "total_questions": "300"}, {"accuracy": "0.74", "provider_name": "PrimeIntellect", "total_questions": "50"}, {"accuracy": "0.7461111666666667", "provider_name": "Reka", "total_questions": "600"}, {"accuracy": "0.76583325", "provider_name": "Relace", "total_questions": "400"}, {"accuracy": "0.7512820769230769", "provider_name": "Sail Research", "total_questions": "650"}, {"accuracy": "0.76", "provider_name": "SiliconFlow", "total_questions": "300"}, {"accuracy": "0.758333375", "provider_name": "StreamLake", "total_questions": "400"}, {"accuracy": "0.737499875", "provider_name": "Together", "total_questions": "400"}, {"accuracy": "0.7558921212121212", "provider_name": "Venice", "total_questions": "396"}, {"accuracy": "0.768", "provider_name": "Wafer", "total_questions": "500"}, {"accuracy": "0.762499875", "provider_name": "Z.AI", "total_questions": "400"}], "rule": "first-party provider (1 of 34 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.42, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.1-2025-11-13_high", "source_row": {"Best score (across scorers)": "0.42", "Country": "United States of America", "Model version": "gpt-5.1-2025-11-13_high", "Organization": "OpenAI", "Release date": "2025-11-13", "Started at": "2026-10-13T14:05:29.000Z", "id": "HKQxqBGokn83tHr5wGJiZX", "mean_score": "0.42", "stderr": "0.04960449637488583"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.19, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.1-2025-11-13_low", "source_row": {"Best score (across scorers)": "0.19", "Country": "United States of America", "Model version": "gpt-5.1-2025-11-13_low", "Organization": "OpenAI", "Release date": "2025-11-13", "Started at": "2026-08-27T18:36:07.000Z", "id": "iV2bSueom3C2SKw72ogiYr", "mean_score": "0.19", "stderr": "0.039427724440366255"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-11-13", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8861111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.1-2025-11-13_high", "source_row": {"Best score (across scorers)": "0.8861111111111111", "Country": "United States of America", "Model version": "gpt-5.1-2025-11-13_high", "Organization": "OpenAI", "Release date": "2025-11-13", "Started at": "2025-11-13T19:52:15.595Z", "id": "mYKKTdV27TRJv9FpUg7rkn", "mean_score": "0.8861111111111111", "stderr": "0.040479727089927024"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-18", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6797520661157025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.1-2025-11-13_high", "source_row": {"Best score (across scorers)": "0.6797520661157025", "Country": "United States of America", "Model version": "gpt-5.1-2025-11-13_high", "Organization": "OpenAI", "Release date": "2025-11-13", "Started at": "2026-02-18T12:36:35.180Z", "id": "YoHtZpMNdsRE6qdj2K4Ucj", "mean_score": "0.6797520661157025", "stderr": "0.02122974956319103"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-11-13", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8762626262626263, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.1-2025-11-13_high", "source_row": {"Best score (across scorers)": "0.8762626262626263", "Country": "United States of America", "Model version": "gpt-5.1-2025-11-13_high", "Organization": "OpenAI", "Release date": "2025-11-13", "Started at": "2025-11-13T19:52:15.039Z", "id": "CSVtMksEVjBNsGJGjW4GcQ", "mean_score": "0.8762626262626263", "stderr": "0.01875368281077595"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8692479999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.1-20251113", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.1-20251113", "rows": [{"accuracy": "0.8686867500000001", "provider_name": "Azure", "total_questions": "792"}, {"accuracy": "0.8692479999999999", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.1-2025-11-13_high", "source_row": {"Best score (across scorers)": "0.48", "Country": "United States of America", "Model version": "gpt-5.1-2025-11-13_high", "Organization": "OpenAI", "Release date": "2025-11-13", "Started at": "2026-08-27T19:28:24.000Z", "id": "QCLnRy69NBH9ip3VYtQneh", "mean_score": "0.48", "stderr": "0.015806639423035177"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7583332500000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.1-20251113", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.1-20251113", "rows": [{"accuracy": "0.7522221666666666", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.7583332500000001", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-02-02", "model": "kimi-k2-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.922, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "fireworks/kimi-k2p5", "source_row": {"Best score (across scorers)": "0.922", "Country": "China", "Model version": "fireworks/kimi-k2p5", "Organization": "Moonshot", "Release date": "2026-01-27", "Started at": "2026-02-02T10:53:14.986Z", "Training compute (FLOP)": "5.8e+24", "id": "mbfZvrSNoXSJqCyD4VKqLe", "mean_score": "0.922", "stderr": "0.039"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-17", "model": "kimi-k2-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.737603305785124, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.5", "source_row": {"Best score (across scorers)": "0.737603305785124", "Country": "China", "Model version": "kimi-k2.5", "Organization": "Moonshot", "Release date": "2026-01-27", "Started at": "2026-02-17T14:43:08.990Z", "Training compute (FLOP)": "5.8e+24", "id": "5wVjUb3yn6bdjPSfPEMuiZ", "mean_score": "0.737603305785124", "stderr": "0.020017832064928274"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-02-02", "model": "kimi-k2-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.876, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "fireworks/kimi-k2p5", "source_row": {"Best score (across scorers)": "0.876", "Country": "China", "Model version": "fireworks/kimi-k2p5", "Organization": "Moonshot", "Release date": "2026-01-27", "Started at": "2026-02-02T11:01:01.260Z", "Training compute (FLOP)": "5.8e+24", "id": "FmqGUq87N4JcuSbo6Uu3U5", "mean_score": "0.876", "stderr": "0.019"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-29", "model": "kimi-k2-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8592592, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2.5-0127", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.5-0127", "rows": [{"accuracy": "0.7937710000000001", "provider_name": "Amazon Bedrock", "total_questions": "1584"}, {"accuracy": "0.82211", "provider_name": "AtlasCloud", "total_questions": "1188"}, {"accuracy": "0.8476576319514662", "provider_name": "DeepInfra", "total_questions": "989"}, {"accuracy": "0.6016879113924051", "provider_name": "DigitalOcean", "total_questions": "790"}, {"accuracy": "0.8592592", "provider_name": "Moonshot AI", "total_questions": "990"}, {"accuracy": "0.8776864563843236", "provider_name": "Novita", "total_questions": "1582"}, {"accuracy": "0.821895294117647", "provider_name": "Phala", "total_questions": "408"}, {"accuracy": "0.873136", "provider_name": "SiliconFlow", "total_questions": "1386"}, {"accuracy": "0.8181818", "provider_name": "StreamLake", "total_questions": "990"}, {"accuracy": "0.8119721658218682", "provider_name": "Venice", "total_questions": "1381"}], "rule": "first-party provider (1 of 10 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "kimi-k2-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.343, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.5", "source_row": {"Best score (across scorers)": "0.343", "Country": "China", "Model version": "kimi-k2.5", "Organization": "Moonshot", "Release date": "2026-01-27", "Started at": "2026-08-27T19:35:45.000Z", "Training compute (FLOP)": "5.8e+24", "id": "RbXXg4JEbTSBnzw3CMSHnD", "mean_score": "0.343", "stderr": "0.015019206922357007"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-29", "model": "kimi-k2-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7277775, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2.5-0127", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.5-0127", "rows": [{"accuracy": "0.6233332500000001", "provider_name": "Amazon Bedrock", "total_questions": "400"}, {"accuracy": "0.6814285714285714", "provider_name": "AtlasCloud", "total_questions": "350"}, {"accuracy": "0.74", "provider_name": "Chutes", "total_questions": "50"}, {"accuracy": "0.7322221666666667", "provider_name": "DeepInfra", "total_questions": "300"}, {"accuracy": "0.7093335999999999", "provider_name": "DigitalOcean", "total_questions": "250"}, {"accuracy": "0.7277775", "provider_name": "Moonshot AI", "total_questions": "300"}, {"accuracy": "0.7222221666666667", "provider_name": "Novita", "total_questions": "300"}, {"accuracy": "0.72", "provider_name": "Phala", "total_questions": "100"}, {"accuracy": "0.698333375", "provider_name": "SiliconFlow", "total_questions": "400"}, {"accuracy": "0.6986667999999999", "provider_name": "StreamLake", "total_questions": "250"}, {"accuracy": "0.7185185555555555", "provider_name": "Venice", "total_questions": "450"}], "rule": "first-party provider (1 of 11 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.7-code", "source_row": {"Best score (across scorers)": "0.14", "Country": "China", "Model version": "kimi-k2.7-code", "Organization": "Moonshot", "Release date": "2026-06-12", "Started at": "2026-10-13T17:13:43.000Z", "id": "bSyTSKD8RiRNZCGNu6MnFp", "mean_score": "0.14", "stderr": "0.0348735088019777"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-13", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12195121951219512, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.7-code", "source_row": {"Best score (across scorers)": "0.12195121951219512", "Country": "China", "Model version": "kimi-k2.7-code", "Organization": "Moonshot", "Release date": "2026-06-12", "Started at": "2026-06-13T01:39:44.000Z", "id": "UTVSgirxQpTgmTHbgE9EUk", "mean_score": "0.12195121951219512", "stderr": "0.051739520574625435"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-13", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5403508771929825, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.7-code", "source_row": {"Best score (across scorers)": "0.5403508771929825", "Country": "China", "Model version": "kimi-k2.7-code", "Organization": "Moonshot", "Release date": "2026-06-12", "Started at": "2026-06-13T01:39:44.000Z", "id": "X3RxRkt2pBUXYJjAQHsUo6", "mean_score": "0.5403508771929825", "stderr": "0.02957276813514758"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.7-code", "source_row": {"Best score (across scorers)": "0.9555555555555556", "Country": "China", "Model version": "kimi-k2.7-code", "Organization": "Moonshot", "Release date": "2026-06-12", "Started at": "2026-08-07T00:20:43.000Z", "id": "ENc5ZM5TpBYNKkTWN4xyQM", "mean_score": "0.9555555555555556", "stderr": "0.031067790907534736"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8787878787878788, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.7-code", "source_row": {"Best score (across scorers)": "0.8787878787878788", "Country": "China", "Model version": "kimi-k2.7-code", "Organization": "Moonshot", "Release date": "2026-06-12", "Started at": "2026-08-07T00:38:29.000Z", "id": "XneEig6pACkAWeekvgCwNk", "mean_score": "0.8787878787878788", "stderr": "0.02325315795194205"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8847767229341582, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2.7-code-20260612", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "rows": [{"accuracy": "0.8442990780261495", "provider_name": "Alibaba", "total_questions": "2371"}, {"accuracy": "0.8419939695817491", "provider_name": "Ambient", "total_questions": "789"}, {"accuracy": "0.836653454183267", "provider_name": "AtlasCloud", "total_questions": "251"}, {"accuracy": "0.8269358", "provider_name": "BaseTen", "total_questions": "990"}, {"accuracy": "0.872399359223301", "provider_name": "Cloudflare", "total_questions": "2163"}, {"accuracy": "0.8751617692307694", "provider_name": "CoreWeave", "total_questions": "2574"}, {"accuracy": "0.7465043071718538", "provider_name": "DeepInfra", "total_questions": "739"}, {"accuracy": "0.8928570861678005", "provider_name": "Fireworks", "total_questions": "882"}, {"accuracy": "0.8613916666666668", "provider_name": "GMICloud", "total_questions": "1782"}, {"accuracy": "0.8875035477214103", "provider_name": "Inceptron", "total_questions": "2326"}, {"accuracy": "0.8727272475524476", "provider_name": "ModelRun", "total_questions": "1430"}, {"accuracy": "0.8847767229341582", "provider_name": "Moonshot AI", "total_questions": "4526"}, {"accuracy": "0.8501683333333333", "provider_name": "Nebius", "total_questions": "1188"}, {"accuracy": "0.8853686363636363", "provider_name": "Novita", "total_questions": "2178"}, {"accuracy": "0.8765007667238421", "provider_name": "Parasail", "total_questions": "583"}, {"accuracy": "0.8821550000000001", "provider_name": "Phala", "total_questions": "396"}, {"accuracy": "0.8783141613793103", "provider_name": "SiliconFlow", "total_questions": "2175"}, {"accuracy": "0.8378130592547341", "provider_name": "StreamLake", "total_questions": "1637"}, {"accuracy": "0.7264309999999999", "provider_name": "Tenstorrent", "total_questions": "396"}, {"accuracy": "0.8571427108013937", "provider_name": "Together", "total_questions": "574"}, {"accuracy": "0.8914076107512954", "provider_name": "Venice", "total_questions": "1544"}], "rule": "first-party provider (1 of 21 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.365, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "kimi-k2.7-code", "source_row": {"Best score (across scorers)": "0.365", "Country": "China", "Model version": "kimi-k2.7-code", "Organization": "Moonshot", "Release date": "2026-06-12", "Started at": "2026-08-27T19:36:03.000Z", "id": "5AijYw5bFJtSuahnbzNo6Z", "mean_score": "0.365", "stderr": "0.015231776226264846"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "kimi-k2-7-code", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7383231627502177, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2.7-code-20260612", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2.7-code-20260612", "rows": [{"accuracy": "0.7360831871227365", "provider_name": "Alibaba", "total_questions": "497"}, {"accuracy": "0.7413334", "provider_name": "Ambient", "total_questions": "250"}, {"accuracy": "0.7341665", "provider_name": "AtlasCloud", "total_questions": "200"}, {"accuracy": "0.593333", "provider_name": "BaseTen", "total_questions": "100"}, {"accuracy": "0.7272220833333334", "provider_name": "Cloudflare", "total_questions": "600"}, {"accuracy": "0.7153846153846154", "provider_name": "CoreWeave", "total_questions": "650"}, {"accuracy": "0.7663313768844222", "provider_name": "DeepInfra", "total_questions": "199"}, {"accuracy": "0.74", "provider_name": "Fireworks", "total_questions": "100"}, {"accuracy": "0.654166625", "provider_name": "GMICloud", "total_questions": "400"}, {"accuracy": "0.7394766996587031", "provider_name": "Inceptron", "total_questions": "586"}, {"accuracy": "0.7328571428571429", "provider_name": "ModelRun", "total_questions": "350"}, {"accuracy": "0.7383231627502177", "provider_name": "Moonshot AI", "total_questions": "1149"}, {"accuracy": "0.7028112570281124", "provider_name": "Nebius", "total_questions": "249"}, {"accuracy": "0.72333325", "provider_name": "Novita", "total_questions": "600"}, {"accuracy": "0.7016665000000001", "provider_name": "Parasail", "total_questions": "200"}, {"accuracy": "0.7333335", "provider_name": "Phala", "total_questions": "100"}, {"accuracy": "0.7283333333333334", "provider_name": "SiliconFlow", "total_questions": "600"}, {"accuracy": "0.7266667142857143", "provider_name": "StreamLake", "total_questions": "350"}, {"accuracy": "0.675", "provider_name": "Tenstorrent", "total_questions": "100"}, {"accuracy": "0.76166675", "provider_name": "Together", "total_questions": "200"}, {"accuracy": "0.7272221666666667", "provider_name": "Venice", "total_questions": "600"}], "rule": "first-party provider (1 of 21 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-15", "model": "inkling-small", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "inkling-small_xhigh", "source_row": {"Best score (across scorers)": "0.06", "Country": "United States of America", "Model version": "inkling-small_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-15T01:02:40.000Z", "id": "nPFA9ufjfVqg3YNmDRqWeC", "mean_score": "0.06", "stderr": "0.023868325657594204"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-14", "model": "inkling-small", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17073170731707318, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "inkling-small_xhigh", "source_row": {"Best score (across scorers)": "0.17073170731707318", "Country": "United States of America", "Model version": "inkling-small_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-14T19:59:09.000Z", "id": "DVD8G8DHNyzkowBzK7JvMt", "mean_score": "0.17073170731707318", "stderr": "0.05949419959829495"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-14", "model": "inkling-small", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4631578947368421, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "inkling-small_xhigh", "source_row": {"Best score (across scorers)": "0.4631578947368421", "Country": "United States of America", "Model version": "inkling-small_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-14T19:59:09.000Z", "id": "B6bZbqickq8o2fQcgDmzMo", "mean_score": "0.4631578947368421", "stderr": "0.02958888847874604"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-14", "model": "inkling-small", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "inkling-small_xhigh", "source_row": {"Best score (across scorers)": "0.9", "Country": "United States of America", "Model version": "inkling-small_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-14T14:19:47.000Z", "id": "53BDFebxiT9KGeP5FRt6bJ", "mean_score": "0.9", "stderr": "0.03964807305493795"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-14", "model": "inkling-small", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.88510101010101, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "inkling-small_xhigh", "source_row": {"Best score (across scorers)": "0.88510101010101", "Country": "United States of America", "Model version": "inkling-small_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-14T14:19:47.000Z", "id": "jTHw5SET8usd7emYPjEDjS", "mean_score": "0.88510101010101", "stderr": "0.019100012831823398"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-09-19", "model": "inkling-small", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8703701666666666, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "thinkingmachines/inkling-small-20260730", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-small-20260730", "rows": [{"accuracy": "0.8703701666666666", "provider_name": "BaseTen", "total_questions": "1188"}, {"accuracy": "0.867634625", "provider_name": "DeepInfra", "total_questions": "1584"}, {"accuracy": "0.8710116425661915", "provider_name": "Together", "total_questions": "982"}], "rule": "median over providers (3 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "inkling-small", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.191, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "inkling-small_xhigh", "source_row": {"Best score (across scorers)": "0.191", "Country": "United States of America", "Model version": "inkling-small_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-27T19:36:17.000Z", "id": "eJfbDKqeg9kNXFoxpeV7HG", "mean_score": "0.191", "stderr": "0.012436787112179482"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-08", "model": "inkling-small", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.6076188571428572, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "thinkingmachines/inkling-small-20260730", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-small-20260730", "rows": [{"accuracy": "0.6076188571428572", "provider_name": "DeepInfra", "total_questions": "350"}, {"accuracy": "0.65166675", "provider_name": "Together", "total_questions": "200"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.45, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-nano-2026-03-17_xhigh", "source_row": {"Best score (across scorers)": "0.45", "Country": "United States of America", "Model version": "gpt-5.4-nano-2026-03-17_xhigh", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-10-14T18:22:26.000Z", "id": "MeFyTsrHFWVrQJ8QHRjnRw", "mean_score": "0.45", "stderr": "0.05"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.09, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-nano-2026-03-17_none", "source_row": {"Best score (across scorers)": "0.09", "Country": "United States of America", "Model version": "gpt-5.4-nano-2026-03-17_none", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-08-27T17:56:52.000Z", "id": "fxNDgLaScEoHo2canW89yv", "mean_score": "0.09", "stderr": "0.028762349126466143"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-12", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12195121951219512, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-nano-2026-03-17_high", "source_row": {"Best score (across scorers)": "0.12195121951219512", "Country": "United States of America", "Model version": "gpt-5.4-nano-2026-03-17_high", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-06-12T16:48:06.000Z", "id": "SF6FYnnKztPCDWQc4v4EJa", "mean_score": "0.12195121951219512", "stderr": "0.051739520574625435"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-12", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.44912280701754387, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-nano-2026-03-17_high", "source_row": {"Best score (across scorers)": "0.44912280701754387", "Country": "United States of America", "Model version": "gpt-5.4-nano-2026-03-17_high", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-06-12T16:48:06.000Z", "id": "iKtopjTjyCWZqJaEjcc7ZL", "mean_score": "0.44912280701754387", "stderr": "0.029515543245521678"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-04-14", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-nano-2026-03-17_high", "source_row": {"Best score (across scorers)": "0.8778", "Country": "United States of America", "Model version": "gpt-5.4-nano-2026-03-17_high", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-04-14T15:08:33.525Z", "id": "jVv9m8DjkdpZaLPdUzAnG9", "mean_score": "0.8778", "stderr": "0.0394"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-04-14", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7847, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-nano-2026-03-17_high", "source_row": {"Best score (across scorers)": "0.7847", "Country": "United States of America", "Model version": "gpt-5.4-nano-2026-03-17_high", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-04-14T15:07:20.513Z", "id": "fZB3qGSohAVYzN97mniuUp", "mean_score": "0.7847", "stderr": "0.0244"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.77567325, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.4-nano-20260317", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-nano-20260317", "rows": [{"accuracy": "0.770202", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.77567325", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.117, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-nano-2026-03-17_high", "source_row": {"Best score (across scorers)": "0.117", "Country": "United States of America", "Model version": "gpt-5.4-nano-2026-03-17_high", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-08-27T19:28:50.000Z", "id": "3QBsxKkbF2R2LEXTd6HDrJ", "mean_score": "0.117", "stderr": "0.010169287802713345"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-4-nano", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.65166675, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.4-nano-20260317", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-nano-20260317", "rows": [{"accuracy": "0.6433335", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.65166675", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-mini-2026-03-17_high", "source_row": {"Best score (across scorers)": "0.23", "Country": "United States of America", "Model version": "gpt-5.4-mini-2026-03-17_high", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-10-13T13:56:43.000Z", "id": "fzgzPmgay5wxMGHjevRjrm", "mean_score": "0.23", "stderr": "0.04229525846816507"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.11, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-mini-2026-03-17_none", "source_row": {"Best score (across scorers)": "0.11", "Country": "United States of America", "Model version": "gpt-5.4-mini-2026-03-17_none", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-08-27T17:57:44.000Z", "id": "NEysw7ag6KJxUahem2PHQv", "mean_score": "0.11", "stderr": "0.03144660377352201"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-12", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0975609756097561, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-mini-2026-03-17_xhigh", "source_row": {"Best score (across scorers)": "0.0975609756097561", "Country": "United States of America", "Model version": "gpt-5.4-mini-2026-03-17_xhigh", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-06-12T16:40:39.000Z", "id": "jtQ6rDiNxjNKkVMEPBE3Ub", "mean_score": "0.0975609756097561", "stderr": "0.04691557088212523"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-12", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.512280701754386, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-mini-2026-03-17_xhigh", "source_row": {"Best score (across scorers)": "0.512280701754386", "Country": "United States of America", "Model version": "gpt-5.4-mini-2026-03-17_xhigh", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-06-12T16:40:38.000Z", "id": "jjFRrQvsud3XsgYufr9f8f", "mean_score": "0.512280701754386", "stderr": "0.02966059084324674"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8888888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-mini-2026-03-17_xhigh", "source_row": {"Best score (across scorers)": "0.8888888888888888", "Country": "United States of America", "Model version": "gpt-5.4-mini-2026-03-17_xhigh", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-08-07T00:56:59.000Z", "id": "QGMk6JNCYuUkFMF4zdVCsM", "mean_score": "0.8888888888888888", "stderr": "0.047377936967913425"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8686868686868687, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-mini-2026-03-17_xhigh", "source_row": {"Best score (across scorers)": "0.8686868686868687", "Country": "United States of America", "Model version": "gpt-5.4-mini-2026-03-17_xhigh", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-08-07T01:00:11.000Z", "id": "gKB2rCD7mHzFN2MCt2YCni", "mean_score": "0.8686868686868687", "stderr": "0.02406315641682249"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.84048825, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.4-mini-20260317", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5.4-mini-20260317", "rows": [{"accuracy": "0.8310885000000001", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.84048825", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.294, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-mini-2026-03-17_high", "source_row": {"Best score (across scorers)": "0.294", "Country": "United States of America", "Model version": "gpt-5.4-mini-2026-03-17_high", "Organization": "OpenAI", "Release date": "2026-03-17", "Started at": "2026-08-27T19:28:49.000Z", "id": "Z3pjKgAULY9j9wmzQpdqZG", "mean_score": "0.294", "stderr": "0.014414290540008265"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-4-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6216667499999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.4-mini-20260317", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.4-mini-20260317", "rows": [{"accuracy": "0.6477778333333334", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.6216667499999999", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.19, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_max", "source_row": {"Best score (across scorers)": "0.19", "Country": "China", "Model version": "deepseek-v4-pro_max", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-10-13T13:38:19.000Z", "Training compute (FLOP)": "9.702e+24", "id": "DSyqeLiA9LLZF4iaxsTVHL", "mean_score": "0.19", "stderr": "0.039427724440366255"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_none", "source_row": {"Best score (across scorers)": "0.17", "Country": "China", "Model version": "deepseek-v4-pro_none", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-08-27T18:01:08.000Z", "Training compute (FLOP)": "9.702e+24", "id": "BmvJaHMCnZzb24ErpAsozU", "mean_score": "0.17", "stderr": "0.03775251680686369"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-17", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.024390243902439025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_max", "source_row": {"Best score (across scorers)": "0.024390243902439025", "Country": "China", "Model version": "deepseek-v4-pro_max", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-06-17T02:30:14.000Z", "Training compute (FLOP)": "9.702e+24", "id": "RAZ6VMfGtQSE8EvUZRRz4Q", "mean_score": "0.024390243902439025", "stderr": "0.024390243902439025"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-17", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.45263157894736844, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_max", "source_row": {"Best score (across scorers)": "0.45263157894736844", "Country": "China", "Model version": "deepseek-v4-pro_max", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-06-17T02:30:14.000Z", "Training compute (FLOP)": "9.702e+24", "id": "KUar3FaFjzq5SMKrEhtn8e", "mean_score": "0.45263157894736844", "stderr": "0.029536098269922092"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-06-17", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_max", "source_row": {"Best score (across scorers)": "0.9666666666666667", "Country": "China", "Model version": "deepseek-v4-pro_max", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-06-17T00:34:52.000Z", "Training compute (FLOP)": "9.702e+24", "id": "HjTDENSpy9sancDzr7pccY", "mean_score": "0.9666666666666667", "stderr": "0.02002208376727653"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-06-18", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7763975155279503, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_max", "source_row": {"Best score (across scorers)": "0.7763975155279503", "Country": "China", "Model version": "deepseek-v4-pro_max", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-06-18T17:43:54.039Z", "Training compute (FLOP)": "9.702e+24", "id": "WUk3f44JhxtAJaSxi6gFZR", "mean_score": "0.7763975155279503", "stderr": "0.01897828052158487"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9090909090909091, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_high", "source_row": {"Best score (across scorers)": "0.9090909090909091", "Country": "China", "Model version": "deepseek-v4-pro_high", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-08-06T23:58:27.000Z", "Training compute (FLOP)": "9.702e+24", "id": "TPdRwHTJtaJYSu2Cxc5Czn", "mean_score": "0.9090909090909091", "stderr": "0.020482086775424225"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-08", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.853535, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-pro-20260423", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "rows": [{"accuracy": "0.8744278861209964", "provider_name": "Alibaba", "total_questions": "1967"}, {"accuracy": "0.8677245714285714", "provider_name": "AtlasCloud", "total_questions": "1386"}, {"accuracy": "0.8597882857142857", "provider_name": "Azure", "total_questions": "1386"}, {"accuracy": "0.8684461428571428", "provider_name": "Baidu", "total_questions": "1386"}, {"accuracy": "0.815286480254777", "provider_name": "BaseTen", "total_questions": "785"}, {"accuracy": "0.8755271341772152", "provider_name": "Cloudflare", "total_questions": "1185"}, {"accuracy": "0.8695285000000001", "provider_name": "CoreWeave", "total_questions": "396"}, {"accuracy": "0.6196141015067407", "provider_name": "DeepInfra", "total_questions": "1261"}, {"accuracy": "0.853535", "provider_name": "DeepSeek", "total_questions": "198"}, {"accuracy": "0.7766553333333335", "provider_name": "DigitalOcean", "total_questions": "1782"}, {"accuracy": "0.8812964663774403", "provider_name": "GMICloud", "total_questions": "1383"}, {"accuracy": "0.8118685", "provider_name": "Ionstream", "total_questions": "792"}, {"accuracy": "0.887537924012158", "provider_name": "NextBit", "total_questions": "987"}, {"accuracy": "0.8852812857142857", "provider_name": "Novita", "total_questions": "1386"}, {"accuracy": "0.8786261545293074", "provider_name": "Parasail", "total_questions": "1126"}, {"accuracy": "0.8702065335018964", "provider_name": "Reka", "total_questions": "791"}, {"accuracy": "0.8845977213793105", "provider_name": "Relace", "total_questions": "725"}, {"accuracy": "0.8559402857142857", "provider_name": "SiliconFlow", "total_questions": "1386"}, {"accuracy": "0.8739477499999999", "provider_name": "StreamLake", "total_questions": "1584"}, {"accuracy": "0.8740044607508533", "provider_name": "Together", "total_questions": "586"}, {"accuracy": "0.7372685", "provider_name": "Venice", "total_questions": "1584"}], "rule": "first-party provider (1 of 21 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46993987975951906, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-v4-pro_max", "source_row": {"Best score (across scorers)": "0.46993987975951906", "Country": "China", "Model version": "deepseek-v4-pro_max", "Organization": "DeepSeek", "Release date": "2026-04-24", "Started at": "2026-08-27T19:30:29.000Z", "Training compute (FLOP)": "9.702e+24", "id": "Sq9xqkVTmFvELpQy9KH5TM", "mean_score": "0.46993987975951906", "stderr": "0.015806515332355317"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-08", "model": "deepseek-v4-pro", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.788889, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-pro-20260423", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-pro-20260423", "rows": [{"accuracy": "0.76777775", "provider_name": "Alibaba", "total_questions": "600"}, {"accuracy": "0.7762962222222223", "provider_name": "AtlasCloud", "total_questions": "450"}, {"accuracy": "0.7647619999999999", "provider_name": "Azure", "total_questions": "350"}, {"accuracy": "0.7718518888888889", "provider_name": "Baidu", "total_questions": "450"}, {"accuracy": "0.7860963903743317", "provider_name": "BaseTen", "total_questions": "187"}, {"accuracy": "0.7704762857142857", "provider_name": "Cloudflare", "total_questions": "350"}, {"accuracy": "0.7684563154362417", "provider_name": "CoreWeave", "total_questions": "149"}, {"accuracy": "0.7800268380566802", "provider_name": "DeepInfra", "total_questions": "247"}, {"accuracy": "0.788889", "provider_name": "DeepSeek", "total_questions": "150"}, {"accuracy": "0.772476937614679", "provider_name": "DigitalOcean", "total_questions": "545"}, {"accuracy": "0.7866667777777777", "provider_name": "GMICloud", "total_questions": "450"}, {"accuracy": "0.765", "provider_name": "Ionstream", "total_questions": "200"}, {"accuracy": "0.7706666", "provider_name": "NextBit", "total_questions": "250"}, {"accuracy": "0.768333375", "provider_name": "Novita", "total_questions": "400"}, {"accuracy": "0.7655824146341464", "provider_name": "Parasail", "total_questions": "246"}, {"accuracy": "0.7826666", "provider_name": "Reka", "total_questions": "250"}, {"accuracy": "0.77083325", "provider_name": "Relace", "total_questions": "200"}, {"accuracy": "0.774074", "provider_name": "SiliconFlow", "total_questions": "450"}, {"accuracy": "0.777037", "provider_name": "StreamLake", "total_questions": "450"}, {"accuracy": "0.7688889999999999", "provider_name": "Venice", "total_questions": "450"}], "rule": "first-party provider (1 of 20 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_medium", "source_row": {"Best score (across scorers)": "0.37", "Country": "United States of America", "Model version": "o3-2025-04-16_medium", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-10-13T14:09:29.000Z", "id": "MiKNm9AZG7MxP6hibnYN8a", "mean_score": "0.37", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.29, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.29", "Country": "United States of America", "Model version": "o3-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-08-27T19:27:05.000Z", "id": "FmLdgAHVM7MPn3JebBvxTb", "mean_score": "0.29", "stderr": "0.04560480215720687"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.3333333333333333", "Country": "United States of America", "Model version": "o3-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-08-27T20:00:08.000Z", "id": "N4HerYebFnJDEU4PEvNNGM", "mean_score": "0.3333333333333333", "stderr": "0.02797271194322297"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-16", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9777190332326284, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.9777190332326284", "Country": "United States of America", "Model version": "o3-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2025-04-16T18:17:06.429Z", "id": "6oW3GceNQznewQJRwLigth", "mean_score": "0.9777190332326284", "stderr": "0.0033230338273894273"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8444444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_medium", "source_row": {"Best score (across scorers)": "0.8444444444444444", "Country": "United States of America", "Model version": "o3-2025-04-16_medium", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-08-07T00:28:27.000Z", "id": "8GjRH3T4wVQMnBKX4fv4ub", "mean_score": "0.8444444444444444", "stderr": "0.05463890236888291"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-06-25", "model": "o3", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.813, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "o3 (high)", "source_row": {"command": "aider --model o3 --reasoning-effort high", "date": "2025-06-25", "dirname": "2025-06-25-21-04-24--o3-price-reduction-high", "model": "o3 (high)", "pass_rate_2": 81.3, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-12", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6231884057971014, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_medium", "source_row": {"Best score (across scorers)": "0.6231884057971014", "Country": "United States of America", "Model version": "o3-2025-04-16_medium", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-02-12T03:46:13.664Z", "id": "AKt7PcE8LQtoMWVuxwMidk", "mean_score": "0.6231884057971014", "stderr": "0.022072335658124498"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-16", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8181818181818182, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.8181818181818182", "Country": "United States of America", "Model version": "o3-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2025-04-16T18:17:09.093Z", "id": "evANZQ9oQTdGYbDERw7VrD", "mean_score": "0.8181818181818182", "stderr": "0.021267123846387202"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "o3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.494, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.494", "Country": "United States of America", "Model version": "o3-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-08-27T19:28:51.000Z", "id": "CxHSa2gY7ySiPPGTZEZaRt", "mean_score": "0.494", "stderr": "0.015818160898606833"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "o3", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.6305, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "o3-2025-04-16 (Prompt)", "source_row": {"Model": "o3-2025-04-16 (Prompt)", "Organization": "OpenAI", "Overall Acc": "63.05%", "Rank": "8"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.1", "source_row": {"Best score (across scorers)": "0.2", "Country": "China", "Model version": "glm-5.1", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-04-07", "Started at": "2026-10-14T06:28:37.000Z", "id": "R7XgJMi2R9sv7tZdx5SZTv", "mean_score": "0.2", "stderr": "0.04020151261036849"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-29", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3684210526315789, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.1", "source_row": {"Best score (across scorers)": "0.3684210526315789", "Country": "China", "Model version": "glm-5.1", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-04-07", "Started at": "2026-08-29T11:06:22.000Z", "id": "Mhwme9odWJMzMjoYkW8r6B", "mean_score": "0.3684210526315789", "stderr": "0.028623772537150215"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-10", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.1", "source_row": {"Best score (across scorers)": "0.9333333333333333", "Country": "China", "Model version": "glm-5.1", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-04-07", "Started at": "2026-08-10T17:44:52.000Z", "id": "b7Lw9ExxAhVY3hdExoh4MP", "mean_score": "0.9333333333333333", "stderr": "0.03760507165451774"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-05-15", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7417, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.1", "source_row": {"Best score (across scorers)": "0.7417", "Country": "China", "Model version": "glm-5.1", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-04-07", "Started at": "2026-05-15T10:07:59.494Z", "id": "glm-5.1-swe-bench-frankenstein", "mean_score": "0.7417", "stderr": "0.0199"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-10", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.898989898989899, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.1", "source_row": {"Best score (across scorers)": "0.898989898989899", "Country": "China", "Model version": "glm-5.1", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-04-07", "Started at": "2026-08-10T17:44:34.000Z", "id": "CBbeMKRRSeRsXnvSfn7Png", "mean_score": "0.898989898989899", "stderr": "0.021469735576055356"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8115756996197718, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.1-20260406", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5.1-20260406", "rows": [{"accuracy": "0.8175460387794024", "provider_name": "Alibaba", "total_questions": "1573"}, {"accuracy": "0.79541113957935", "provider_name": "AtlasCloud", "total_questions": "1569"}, {"accuracy": "0.7554704302103251", "provider_name": "Baidu", "total_questions": "1569"}, {"accuracy": "0.7991169867549668", "provider_name": "Chutes", "total_questions": "1359"}, {"accuracy": "0.5872781331360947", "provider_name": "CoreWeave", "total_questions": "338"}, {"accuracy": "0.8261094014447884", "provider_name": "Crusoe", "total_questions": "969"}, {"accuracy": "0.7774583256704981", "provider_name": "DeepInfra", "total_questions": "522"}, {"accuracy": "0.6552547006369427", "provider_name": "DigitalOcean", "total_questions": "628"}, {"accuracy": "0.8337731531353135", "provider_name": "Friendli", "total_questions": "1515"}, {"accuracy": "0.7606508265895953", "provider_name": "GMICloud", "total_questions": "1557"}, {"accuracy": "0.7660061679586564", "provider_name": "Nebius", "total_questions": "1161"}, {"accuracy": "0.8098576959847037", "provider_name": "Novita", "total_questions": "1569"}, {"accuracy": "0.8477753864168618", "provider_name": "Parasail", "total_questions": "427"}, {"accuracy": "0.7979798333333333", "provider_name": "Phala", "total_questions": "1188"}, {"accuracy": "0.8222643415559772", "provider_name": "SiliconFlow", "total_questions": "1581"}, {"accuracy": "0.8360502124789209", "provider_name": "StreamLake", "total_questions": "1779"}, {"accuracy": "0.7948342770467838", "provider_name": "Venice", "total_questions": "1368"}, {"accuracy": "0.8115756996197718", "provider_name": "Z.AI", "total_questions": "1578"}], "rule": "first-party provider (1 of 18 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5.1", "source_row": {"Best score (across scorers)": "0.34", "Country": "China", "Model version": "glm-5.1", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-04-07", "Started at": "2026-08-27T19:36:21.000Z", "id": "R2fMFAtRXah864hMifdQGV", "mean_score": "0.34", "stderr": "0.014987482264363972"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "glm-5-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7433332500000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5.1-20260406", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5.1-20260406", "rows": [{"accuracy": "0.76958325", "provider_name": "Alibaba", "total_questions": "400"}, {"accuracy": "0.7481481111111111", "provider_name": "AtlasCloud", "total_questions": "450"}, {"accuracy": "0.742083375", "provider_name": "Baidu", "total_questions": "400"}, {"accuracy": "0.736666625", "provider_name": "Chutes", "total_questions": "400"}, {"accuracy": "0.753333", "provider_name": "CoreWeave", "total_questions": "50"}, {"accuracy": "0.7583331666666667", "provider_name": "Crusoe", "total_questions": "300"}, {"accuracy": "0.7411113333333332", "provider_name": "DeepInfra", "total_questions": "150"}, {"accuracy": "0.7421122716049383", "provider_name": "DigitalOcean", "total_questions": "243"}, {"accuracy": "0.670000125", "provider_name": "Friendli", "total_questions": "400"}, {"accuracy": "0.735000125", "provider_name": "Nebius", "total_questions": "400"}, {"accuracy": "0.729166625", "provider_name": "Novita", "total_questions": "400"}, {"accuracy": "0.7666669999999999", "provider_name": "Parasail", "total_questions": "50"}, {"accuracy": "0.752381", "provider_name": "Phala", "total_questions": "350"}, {"accuracy": "0.7503703333333334", "provider_name": "SiliconFlow", "total_questions": "450"}, {"accuracy": "0.7474074444444444", "provider_name": "StreamLake", "total_questions": "450"}, {"accuracy": "0.7483332500000001", "provider_name": "Venice", "total_questions": "400"}, {"accuracy": "0.7433332500000001", "provider_name": "Z.AI", "total_questions": "400"}], "rule": "first-party provider (1 of 17 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.24, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.24", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-10-13T14:09:48.000Z", "id": "5tdGvpe8Ahh8xsUnSwhdAF", "mean_score": "0.24", "stderr": "0.04292346959909278"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_minimal", "source_row": {"Best score (across scorers)": "0.1", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_minimal", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-08-27T17:58:37.000Z", "id": "kW6hSbkCwQczqbazduGXPR", "mean_score": "0.1", "stderr": "0.03015113445777636"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-12", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12195121951219512, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.12195121951219512", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-06-12T16:48:15.000Z", "id": "2WxRbjq5ZWTgiXZPqYUG4R", "mean_score": "0.12195121951219512", "stderr": "0.051739520574625435"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-12", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.4666666666666667", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-06-12T16:48:15.000Z", "id": "YSR9WCR2SnJYvP7DJwX7YN", "mean_score": "0.4666666666666667", "stderr": "0.029603535719125714"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-10-30", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9784743202416919, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.9784743202416919", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-30T10:00:02.998Z", "id": "5duoiNDccsgaW7UVt6feYX", "mean_score": "0.9784743202416919", "stderr": "0.003369422414145465"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-30", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.8666666666666667", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-30T09:59:54.926Z", "id": "R82gycJfm5jaCdh32tFkH4", "mean_score": "0.8666666666666667", "stderr": "0.04286165199944133"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-01", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6466942148760331, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_medium", "source_row": {"Best score (across scorers)": "0.6466942148760331", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_medium", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-02-01T11:28:37.125Z", "id": "7nvdQy3AwKtoay2FBvfST5", "mean_score": "0.6466942148760331", "stderr": "0.021749601139671306"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-10-30", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.75, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.75", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-30T09:59:56.459Z", "id": "jP8EyLtPFsakgnsHe2oJM6", "mean_score": "0.75", "stderr": "0.022767295427113803"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8007853333333332, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5-mini-2025-08-07", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "rows": [{"accuracy": "0.7996631666666666", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.8007853333333332", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.216, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-mini-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.216", "Country": "United States of America", "Model version": "gpt-5-mini-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-08-10T21:35:29.000Z", "id": "8d5RuFTTJKuJ5iJYmPZVJe", "mean_score": "0.216", "stderr": "0.013019735539307761"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5546, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "GPT-5-mini-2025-08-07 (FC)", "source_row": {"Model": "GPT-5-mini-2025-08-07 (FC)", "Organization": "OpenAI", "Overall Acc": "55.46%", "Rank": "17"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.712121101010101, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5-mini-2025-08-07", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-mini-2025-08-07", "rows": [{"accuracy": "0.701111", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.712121101010101", "provider_name": "OpenAI", "total_questions": "198"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen3-6-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.07, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-27b_none", "source_row": {"Best score (across scorers)": "0.07", "Country": "China", "Model version": "qwen3.6-27b_none", "Organization": "Alibaba", "Release date": "2026-04-22", "Started at": "2026-08-27T18:48:08.000Z", "id": "dKZDn279PKuc2Kxm68D4r2", "mean_score": "0.07", "stderr": "0.02564323999762428"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-28", "model": "qwen3-6-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3508771929824561, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-27b", "source_row": {"Best score (across scorers)": "0.3508771929824561", "Country": "China", "Model version": "qwen3.6-27b", "Organization": "Alibaba", "Release date": "2026-04-22", "Started at": "2026-08-28T12:30:22.000Z", "id": "VonoEhAsGSJ2FHU33qnhui", "mean_score": "0.3508771929824561", "stderr": "0.028319255656547093"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen3-6-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-27b", "source_row": {"Best score (across scorers)": "0.9111111111111111", "Country": "China", "Model version": "qwen3.6-27b", "Organization": "Alibaba", "Release date": "2026-04-22", "Started at": "2026-08-07T00:12:36.000Z", "id": "452pB5w5ea6uy92e2q2NMv", "mean_score": "0.9111111111111111", "stderr": "0.04290254662948546"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen3-6-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8585858585858586, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-27b", "source_row": {"Best score (across scorers)": "0.8585858585858586", "Country": "China", "Model version": "qwen3.6-27b", "Organization": "Alibaba", "Release date": "2026-04-22", "Started at": "2026-08-07T00:24:27.000Z", "id": "buLf8Ne29xmzMZBRQkvrQG", "mean_score": "0.8585858585858586", "stderr": "0.024825909793343353"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "qwen3-6-27b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.801499818181818, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.6-27b-20260422", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-27b-20260422", "rows": [{"accuracy": "0.801499818181818", "provider_name": "Alibaba", "total_questions": "2178"}, {"accuracy": "0.8365635357637288", "provider_name": "Chutes", "total_questions": "2167"}, {"accuracy": "0.828497570984456", "provider_name": "CoreWeave", "total_questions": "965"}, {"accuracy": "0.8340288847157016", "provider_name": "DeepInfra", "total_questions": "1917"}, {"accuracy": "0.7112403023255814", "provider_name": "Io Net", "total_questions": "344"}, {"accuracy": "0.7508415000000002", "provider_name": "Morph", "total_questions": "396"}, {"accuracy": "0.7893355839160839", "provider_name": "Phala", "total_questions": "1716"}, {"accuracy": "0.8050312243186584", "provider_name": "Reka", "total_questions": "477"}, {"accuracy": "0.8327431107738998", "provider_name": "SiliconFlow", "total_questions": "1977"}, {"accuracy": "0.8505509090909091", "provider_name": "Venice", "total_questions": "2178"}], "rule": "first-party provider (1 of 10 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "qwen3-6-27b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7627176144578314, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.6-27b-20260422", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-27b-20260422", "rows": [{"accuracy": "0.7627176144578314", "provider_name": "Alibaba", "total_questions": "498"}, {"accuracy": "0.7863944591836735", "provider_name": "Chutes", "total_questions": "490"}, {"accuracy": "0.7626666", "provider_name": "CoreWeave", "total_questions": "250"}, {"accuracy": "0.7342658006993007", "provider_name": "DeepInfra", "total_questions": "286"}, {"accuracy": "0.7088886666666667", "provider_name": "Io Net", "total_questions": "150"}, {"accuracy": "0.7454957162162162", "provider_name": "Morph", "total_questions": "148"}, {"accuracy": "0.7133782062780268", "provider_name": "Phala", "total_questions": "446"}, {"accuracy": "0.7350427912087912", "provider_name": "SiliconFlow", "total_questions": "546"}, {"accuracy": "0.7417582051282051", "provider_name": "Venice", "total_questions": "546"}], "rule": "first-party provider (1 of 9 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen3-7-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-plus_none", "source_row": {"Best score (across scorers)": "0.17", "Country": "China", "Model version": "qwen3.7-plus_none", "Organization": "Alibaba", "Release date": "2026-06-02", "Started at": "2026-08-27T18:18:28.000Z", "id": "a5qR5FNnftfR8pk9HJ3JUz", "mean_score": "0.17", "stderr": "0.03775251680686369"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-29", "model": "qwen3-7-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34385964912280703, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-plus_none", "source_row": {"Best score (across scorers)": "0.34385964912280703", "Country": "China", "Model version": "qwen3.7-plus_none", "Organization": "Alibaba", "Release date": "2026-06-02", "Started at": "2026-08-29T11:07:26.000Z", "id": "Tt8tBXHoCEYqGLePWXkM8u", "mean_score": "0.34385964912280703", "stderr": "0.028185763989072583"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen3-7-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-plus", "source_row": {"Best score (across scorers)": "0.9333333333333333", "Country": "China", "Model version": "qwen3.7-plus", "Organization": "Alibaba", "Release date": "2026-06-02", "Started at": "2026-08-07T00:02:57.000Z", "id": "Jvdw6GAXiS6T79bkpbk6Ra", "mean_score": "0.9333333333333333", "stderr": "0.03760507165451775"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen3-7-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8787878787878788, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-plus", "source_row": {"Best score (across scorers)": "0.8787878787878788", "Country": "China", "Model version": "qwen3.7-plus", "Organization": "Alibaba", "Release date": "2026-06-02", "Started at": "2026-08-07T00:07:25.000Z", "id": "URoWjw9Htc6QPwABVcrybw", "mean_score": "0.8787878787878788", "stderr": "0.02325315795194205"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "qwen3-7-plus", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.873737, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.7-plus-20260602", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.7-plus-20260602", "rows": [{"accuracy": "0.873737", "provider_name": "Alibaba", "total_questions": "198"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "qwen3-7-plus", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.726667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.7-plus-20260602", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.7-plus-20260602", "rows": [{"accuracy": "0.726667", "provider_name": "Alibaba", "total_questions": "50"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_59K", "source_row": {"Best score (across scorers)": "0.14", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_59K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2026-10-13T13:42:53.000Z", "id": "DPuTDKvupUqKgkdVj7Awsp", "mean_score": "0.14", "stderr": "0.0348735088019777"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2833333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_64K", "source_row": {"Best score (across scorers)": "0.2833333333333333", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_64K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2026-09-10T20:22:43.000Z", "id": "E3gZb5amR7rcoRkqUGeQRG", "mean_score": "0.2833333333333333", "stderr": "0.05866531842313124"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-25", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_48K", "source_row": {"Best score (across scorers)": "0.22", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_48K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2026-07-25T07:05:27.000Z", "id": "k4zGHaXESxFRTRMkyDLZrz", "mean_score": "0.22", "stderr": "0.04163331998932266"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.04878048780487805, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_32K", "source_row": {"Best score (across scorers)": "0.04878048780487805", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_32K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2026-06-11T01:55:25.000Z", "id": "azk4theXxAsizmoi2HDD49", "mean_score": "0.04878048780487805", "stderr": "0.03405912205797303"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34385964912280703, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_32K", "source_row": {"Best score (across scorers)": "0.34385964912280703", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_32K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2026-06-11T01:55:25.000Z", "id": "mS5EwCRZVpvUvJEehPUDZu", "mean_score": "0.34385964912280703", "stderr": "0.028185763989072587"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-11-24", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8611111111111112, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_32K", "source_row": {"Best score (across scorers)": "0.8611111111111112", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_32K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2025-11-24T20:14:50.875Z", "id": "ikUWdu6HJ2qRgZynfzDCFe", "mean_score": "0.8611111111111112", "stderr": "0.042644255740946095"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-05", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7665289256198347, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101", "source_row": {"Best score (across scorers)": "0.7665289256198347", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2026-02-05T22:06:33.964Z", "id": "mKQKxkpGDrFHBPBYR3mBrB", "mean_score": "0.7665289256198347", "stderr": "0.01924895909142592"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-11-24", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.860479797979798, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_32K", "source_row": {"Best score (across scorers)": "0.860479797979798", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_32K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2025-11-24T20:14:34.565Z", "id": "Kwwo3UFcPDk6iLAeSeeRaa", "mean_score": "0.860479797979798", "stderr": "0.020686959671393914"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8610461538461541, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.5-opus-20251124", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "rows": [{"accuracy": "0.8610345454545455", "provider_name": "Amazon Bedrock", "total_questions": "4356"}, {"accuracy": "0.8610461538461541", "provider_name": "Anthropic", "total_questions": "2574"}, {"accuracy": "0.8562900909090908", "provider_name": "Azure", "total_questions": "2178"}, {"accuracy": "0.8569022", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.859963090909091", "provider_name": "Google", "total_questions": "2178"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.457, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-5-20251101_32K", "source_row": {"Best score (across scorers)": "0.457", "Country": "United States of America", "Model version": "claude-opus-4-5-20251101_32K", "Organization": "Anthropic", "Release date": "2025-11-24", "Started at": "2026-08-27T19:28:02.000Z", "id": "CA5LW4phRnHSsZUGXEm8H6", "mean_score": "0.457", "stderr": "0.0157606915901365"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.7746999999999999, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Claude-Opus-4-5-20251101 (FC)", "source_row": {"Model": "Claude-Opus-4-5-20251101 (FC)", "Organization": "Anthropic", "Overall Acc": "77.47%", "Rank": "1"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-opus-4-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7700000714285714, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.5-opus-20251124", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-opus-20251124", "rows": [{"accuracy": "0.7684848636363636", "provider_name": "Amazon Bedrock", "total_questions": "1100"}, {"accuracy": "0.7700000714285714", "provider_name": "Anthropic", "total_questions": "700"}, {"accuracy": "0.7557576363636364", "provider_name": "Azure", "total_questions": "550"}, {"accuracy": "0.7637036666666667", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7636362727272727", "provider_name": "Google", "total_questions": "550"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "inkling", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Inkling_xhigh", "source_row": {"Best score (across scorers)": "0.21", "Country": "United States of America", "Model version": "Inkling_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-10-13T12:49:23.000Z", "Training compute (FLOP)": "1.1e+25", "id": "UZ9LtVBCjHPMN9SKFUo7Wa", "mean_score": "0.21", "stderr": "0.040936018074033236"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-06", "model": "inkling", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.04878048780487805, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Inkling_xhigh", "source_row": {"Best score (across scorers)": "0.04878048780487805", "Country": "United States of America", "Model version": "Inkling_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-06T20:14:32.000Z", "Training compute (FLOP)": "1.1e+25", "id": "jvJPoRYkPH9Sb5MX252oHp", "mean_score": "0.04878048780487805", "stderr": "0.03405912205797303"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-06", "model": "inkling", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Inkling_xhigh", "source_row": {"Best score (across scorers)": "0.3333333333333333", "Country": "United States of America", "Model version": "Inkling_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-06T20:14:32.000Z", "Training compute (FLOP)": "1.1e+25", "id": "6KvQvuMMX5zAuteUc6pFcW", "mean_score": "0.3333333333333333", "stderr": "0.02797271194322297"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-05", "model": "inkling", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8888888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Inkling_xhigh", "source_row": {"Best score (across scorers)": "0.8888888888888888", "Country": "United States of America", "Model version": "Inkling_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-05T18:55:39.000Z", "Training compute (FLOP)": "1.1e+25", "id": "j2fELMf364qfh2cpaVDL6W", "mean_score": "0.8888888888888888", "stderr": "0.039171949640122654"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-05", "model": "inkling", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8825757575757576, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Inkling_xhigh", "source_row": {"Best score (across scorers)": "0.8825757575757576", "Country": "United States of America", "Model version": "Inkling_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-05T18:55:39.000Z", "Training compute (FLOP)": "1.1e+25", "id": "dfmiZmjAcNkLuAJL4z65eb", "mean_score": "0.8825757575757576", "stderr": "0.019355903133033585"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-09-19", "model": "inkling", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.818035862576065, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "thinkingmachines/inkling-20260715", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "thinkingmachines/inkling-20260715", "rows": [{"accuracy": "0.818035862576065", "provider_name": "BaseTen", "run_count": "10", "total_questions": "1972"}, {"accuracy": "0.8425437868421053", "provider_name": "Together", "run_count": "6", "total_questions": "1140"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "inkling", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.403, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Inkling_xhigh", "source_row": {"Best score (across scorers)": "0.403", "Country": "United States of America", "Model version": "Inkling_xhigh", "Organization": "Thinking Machines", "Release date": "2026-07-15", "Started at": "2026-08-27T19:36:45.000Z", "Training compute (FLOP)": "1.1e+25", "id": "bpnzC6CCi3hWuKCfqX4KEY", "mean_score": "0.403", "stderr": "0.015518757419066513"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-08", "model": "inkling", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6755556666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "thinkingmachines/inkling-20260715", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "thinkingmachines/inkling-20260715", "rows": [{"accuracy": "0.6655251643835617", "provider_name": "BaseTen", "total_questions": "146"}, {"accuracy": "0.6755556666666667", "provider_name": "DeepInfra", "total_questions": "150"}, {"accuracy": "0.6813334", "provider_name": "Together", "total_questions": "250"}], "rule": "median over providers (3 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen3-5-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.18, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-397b-a17b_none", "source_row": {"Best score (across scorers)": "0.18", "Country": "China", "Model version": "qwen3.5-397b-a17b_none", "Organization": "Alibaba", "Release date": "2026-02-13", "Started at": "2026-08-27T18:47:58.000Z", "id": "D7WDyCr24AVdzr4eKAreJu", "mean_score": "0.18", "stderr": "0.03861229196653691"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-30", "model": "qwen3-5-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.312280701754386, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-397b-a17b_none", "source_row": {"Best score (across scorers)": "0.312280701754386", "Country": "China", "Model version": "qwen3.5-397b-a17b_none", "Organization": "Alibaba", "Release date": "2026-02-13", "Started at": "2026-08-30T22:34:18.000Z", "id": "A4yUA9PfzRr92JAqDDo9ro", "mean_score": "0.312280701754386", "stderr": "0.027499133473298815"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen3-5-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8888888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-397b-a17b", "source_row": {"Best score (across scorers)": "0.8888888888888888", "Country": "China", "Model version": "qwen3.5-397b-a17b", "Organization": "Alibaba", "Release date": "2026-02-13", "Started at": "2026-08-07T00:12:18.000Z", "id": "JYuFdyZZSiMvmSZNzMTywb", "mean_score": "0.8888888888888888", "stderr": "0.04737793696791343"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen3-5-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8636363636363636, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-397b-a17b_none", "source_row": {"Best score (across scorers)": "0.8636363636363636", "Country": "China", "Model version": "qwen3.5-397b-a17b_none", "Organization": "Alibaba", "Release date": "2026-02-13", "Started at": "2026-08-07T00:24:02.000Z", "id": "bXrmQqzF9sXbuZfRrSc7NP", "mean_score": "0.8636363636363636", "stderr": "0.02445015597318985"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "qwen3-5-397b-a17b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8728152090966519, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-397b-a17b-20260216", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "rows": [{"accuracy": "0.8728152090966519", "provider_name": "Alibaba", "total_questions": "1583"}, {"accuracy": "0.8648990000000001", "provider_name": "AtlasCloud", "total_questions": "1584"}, {"accuracy": "0.8723343333333334", "provider_name": "Chutes", "total_questions": "594"}, {"accuracy": "0.8699560401315789", "provider_name": "DeepInfra", "total_questions": "1520"}, {"accuracy": "0.8045648025477706", "provider_name": "DigitalOcean", "total_questions": "1570"}, {"accuracy": "0.877104", "provider_name": "GMICloud", "total_questions": "198"}, {"accuracy": "0.8637523318442885", "provider_name": "Novita", "total_questions": "1567"}, {"accuracy": "0.882724073089701", "provider_name": "Parasail", "total_questions": "1505"}, {"accuracy": "0.87542075", "provider_name": "Phala", "total_questions": "1584"}, {"accuracy": "0.8614476", "provider_name": "StreamLake", "total_questions": "1980"}, {"accuracy": "0.836700375", "provider_name": "Venice", "total_questions": "1584"}], "rule": "first-party provider (1 of 11 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "qwen3-5-397b-a17b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7811111111111111, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-397b-a17b-20260216", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-397b-a17b-20260216", "rows": [{"accuracy": "0.7811111111111111", "provider_name": "Alibaba", "total_questions": "450"}, {"accuracy": "0.7546065175879397", "provider_name": "AtlasCloud", "total_questions": "398"}, {"accuracy": "0.7162160675675675", "provider_name": "Chutes", "total_questions": "148"}, {"accuracy": "0.7630522329317269", "provider_name": "DeepInfra", "total_questions": "249"}, {"accuracy": "0.7936708987341772", "provider_name": "DigitalOcean", "total_questions": "395"}, {"accuracy": "0.773333", "provider_name": "GMICloud", "total_questions": "50"}, {"accuracy": "0.7378469999999999", "provider_name": "Novita", "total_questions": "96"}, {"accuracy": "0.773639194842407", "provider_name": "Parasail", "total_questions": "349"}, {"accuracy": "0.7897823718592966", "provider_name": "Phala", "total_questions": "398"}, {"accuracy": "0.7548148888888889", "provider_name": "Venice", "total_questions": "450"}], "rule": "first-party provider (1 of 10 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-6_max", "source_row": {"Best score (across scorers)": "0.23", "Country": "United States of America", "Model version": "claude-sonnet-4-6_max", "Organization": "Anthropic", "Release date": "2026-02-17", "Started at": "2026-10-13T13:44:15.000Z", "id": "6P9AJQmBzaPmE7ossbqLjz", "mean_score": "0.23", "stderr": "0.04229525846816507"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-06", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.16, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-6_low", "source_row": {"Best score (across scorers)": "0.16", "Country": "United States of America", "Model version": "claude-sonnet-4-6_low", "Organization": "Anthropic", "Release date": "2026-02-17", "Started at": "2026-08-06T00:18:14.000Z", "id": "YdyvoMQ4mA5V8Ymg6Tksxu", "mean_score": "0.16", "stderr": "0.03684529491774706"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-02-20", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.857936507936508, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-6_32K", "source_row": {"Best score (across scorers)": "0.857936507936508", "Country": "United States of America", "Model version": "claude-sonnet-4-6_32K", "Organization": "Anthropic", "Release date": "2026-02-17", "Started at": "2026-02-20T16:18:28.678Z", "id": "9LWdzVN5w4ihNvncsaCWMu", "mean_score": "0.857936507936508", "stderr": "0.04539927553202047"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-21", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7520661157024794, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-6", "source_row": {"Best score (across scorers)": "0.7520661157024794", "Country": "United States of America", "Model version": "claude-sonnet-4-6", "Organization": "Anthropic", "Release date": "2026-02-17", "Started at": "2026-02-21T10:32:51.288Z", "id": "KEdgiMmtJKHxgGiS82agYj", "mean_score": "0.7520661157024794", "stderr": "0.01964818270139032"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-02-20", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8737373737373737, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-6_32K", "source_row": {"Best score (across scorers)": "0.8737373737373737", "Country": "United States of America", "Model version": "claude-sonnet-4-6_32K", "Organization": "Anthropic", "Release date": "2026-02-17", "Started at": "2026-02-20T17:51:37.062Z", "id": "58eQmyCfPa3FufhXJFvAL2", "mean_score": "0.8737373737373737", "stderr": "0.020034282226331888"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6459467692307692, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.6-sonnet-20260217", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "rows": [{"accuracy": "0.6708499999999998", "provider_name": "Amazon Bedrock", "total_questions": "6534"}, {"accuracy": "0.6459467692307692", "provider_name": "Anthropic", "total_questions": "2574"}, {"accuracy": "0.6672790000000001", "provider_name": "Azure", "total_questions": "2178"}, {"accuracy": "0.6814815", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.6714038275698719", "provider_name": "Google", "total_questions": "6333"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.355, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-6_high", "source_row": {"Best score (across scorers)": "0.355", "Country": "United States of America", "Model version": "claude-sonnet-4-6_high", "Organization": "Anthropic", "Release date": "2026-02-17", "Started at": "2026-08-10T22:42:23.000Z", "id": "LJyKf8Xy4dymxNHbGLwNdA", "mean_score": "0.355", "stderr": "0.015139491543780595"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-sonnet-4-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7490475714285715, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.6-sonnet-20260217", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.6-sonnet-20260217", "rows": [{"accuracy": "0.75416675", "provider_name": "Amazon Bedrock", "total_questions": "1600"}, {"accuracy": "0.7490475714285715", "provider_name": "Anthropic", "total_questions": "700"}, {"accuracy": "0.7545454545454545", "provider_name": "Azure", "total_questions": "550"}, {"accuracy": "0.7481482222222222", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7453763225806453", "provider_name": "Google", "total_questions": "1550"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "glm-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7", "source_row": {"Best score (across scorers)": "0.21", "Country": "China", "Model version": "glm-4.7", "Organization": "Z.ai (Zhipu AI)", "Release date": "2025-12-22", "Started at": "2026-10-14T09:36:09.000Z", "Training compute (FLOP)": "4.42e+24", "id": "9rDLegdh3zmBfFzTj3fkCc", "mean_score": "0.21", "stderr": "0.040936018074033236"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-01-29", "model": "glm-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8333333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7", "source_row": {"Best score (across scorers)": "0.8333333333333334", "Country": "China", "Model version": "glm-4.7", "Organization": "Z.ai (Zhipu AI)", "Release date": "2025-12-22", "Started at": "2026-01-29T16:46:50.435Z", "Training compute (FLOP)": "4.42e+24", "id": "GYzmrzMfeSjNjm2J6L9PeT", "mean_score": "0.8333333333333334", "stderr": "0.04767312946227961"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-01-29", "model": "glm-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8333333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7", "source_row": {"Best score (across scorers)": "0.8333333333333334", "Country": "China", "Model version": "glm-4.7", "Organization": "Z.ai (Zhipu AI)", "Release date": "2025-12-22", "Started at": "2026-01-29T20:49:26.702Z", "Training compute (FLOP)": "4.42e+24", "id": "6phXzd4aCbnBUSQmBsaywa", "mean_score": "0.8333333333333334", "stderr": "0.02374901666263655"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "glm-4-7", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8364136921606119, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-4.7-20251222", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-20251222", "rows": [{"accuracy": "0.8319781768292682", "provider_name": "AtlasCloud", "total_questions": "984"}, {"accuracy": "0.48091999999999996", "provider_name": "Cerebras", "total_questions": "594"}, {"accuracy": "0.8035647794928336", "provider_name": "DeepInfra", "total_questions": "907"}, {"accuracy": "0.826809625", "provider_name": "Google", "total_questions": "1584"}, {"accuracy": "0.7269411348511384", "provider_name": "Mancer 2", "total_questions": "1142"}, {"accuracy": "0.8452150751445088", "provider_name": "Novita", "total_questions": "1557"}, {"accuracy": "0.8283759292307693", "provider_name": "StreamLake", "total_questions": "975"}, {"accuracy": "0.6374194522580645", "provider_name": "Venice", "total_questions": "1550"}, {"accuracy": "0.8364136921606119", "provider_name": "Z.AI", "total_questions": "1569"}], "rule": "first-party provider (1 of 9 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "glm-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.322, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7", "source_row": {"Best score (across scorers)": "0.322", "Country": "China", "Model version": "glm-4.7", "Organization": "Z.ai (Zhipu AI)", "Release date": "2025-12-22", "Started at": "2026-08-27T19:36:02.000Z", "Training compute (FLOP)": "4.42e+24", "id": "aYau6EG7CCaWVSyVq3vvQc", "mean_score": "0.322", "stderr": "0.014782913600996744"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "glm-4-7", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7037037777777777, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-4.7-20251222", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-20251222", "rows": [{"accuracy": "0.7422221666666666", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.74", "provider_name": "Cerebras", "total_questions": "200"}, {"accuracy": "0.745", "provider_name": "DeepInfra", "total_questions": "300"}, {"accuracy": "0.7481398080357142", "provider_name": "Google", "total_questions": "448"}, {"accuracy": "0.493333", "provider_name": "Mancer 2", "total_questions": "50"}, {"accuracy": "0.7311112222222222", "provider_name": "Novita", "total_questions": "450"}, {"accuracy": "0.7144445", "provider_name": "StreamLake", "total_questions": "300"}, {"accuracy": "0.7414815555555555", "provider_name": "Venice", "total_questions": "450"}, {"accuracy": "0.7037037777777777", "provider_name": "Z.AI", "total_questions": "450"}], "rule": "first-party provider (1 of 9 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_medium", "source_row": {"Best score (across scorers)": "0.22", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_medium", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-10-13T14:10:20.000Z", "id": "dfonReEmxqpRWn5kxGSSES", "mean_score": "0.22", "stderr": "0.041633319989322654"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.05, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.05", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-08-27T19:26:53.000Z", "id": "eJbFTXaxZjnW7HNR93u3WH", "mean_score": "0.05", "stderr": "0.02190429135575903"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.04878048780487805, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.04878048780487805", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-06-11T13:02:32.385Z", "id": "TomkQucNryhFGkaUnUGeLR", "mean_score": "0.04878048780487805", "stderr": "0.03405912205797303"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.36140350877192984, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.36140350877192984", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-06-11T13:04:19.278Z", "id": "dcmRi5JTa27EuwiQTdqr2B", "mean_score": "0.36140350877192984", "stderr": "0.028506918644975"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-16", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.978285498489426, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.978285498489426", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2025-04-16T18:17:08.504Z", "id": "727ixGKZ7A7G8LmavE2ScL", "mean_score": "0.978285498489426", "stderr": "0.0033204985518619605"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-16", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8166666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.8166666666666667", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2025-04-16T18:17:08.217Z", "id": "TVsxkJ8ZyfMVmVx88Qq3KN", "mean_score": "0.8166666666666667", "stderr": "0.046939139620180814"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-16", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.72, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "o4-mini (high)", "source_row": {"command": "aider --model o4-mini", "date": "2025-04-16", "dirname": "2025-04-16-22-01-58--o4-mini-high-diff-exsys", "model": "o4-mini (high)", "pass_rate_2": 72.0, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-16", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7960858585858586, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_high", "source_row": {"Best score (across scorers)": "0.7960858585858586", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_high", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2025-04-16T18:17:16.139Z", "id": "9Sgsz5XCjscuZkexf7MX9c", "mean_score": "0.7960858585858586", "stderr": "0.024034764096121958"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.196, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o4-mini-2025-04-16_low", "source_row": {"Best score (across scorers)": "0.196", "Country": "United States of America", "Model version": "o4-mini-2025-04-16_low", "Organization": "OpenAI", "Release date": "2025-04-16", "Started at": "2026-08-27T19:29:17.000Z", "id": "Dzky9brdTFbSoVd2zTKiUp", "mean_score": "0.196", "stderr": "0.012559527926707347"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "o4-mini", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5324, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "o4-mini-2025-04-16 (FC)", "source_row": {"Model": "o4-mini-2025-04-16 (FC)", "Organization": "OpenAI", "Overall Acc": "53.24%", "Rank": "21"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen3-235b-a22b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.09, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-235B-A22B-Thinking-2507", "source_row": {"Best score (across scorers)": "0.09", "Country": "China", "Model version": "Qwen3-235B-A22B-Thinking-2507", "Organization": "Alibaba", "Release date": "2025-07-25", "Started at": "2026-08-27T17:58:05.000Z", "Training compute (FLOP)": "4.752e+24", "id": "BP3KYD8rYyFg3LviNGtjLL", "mean_score": "0.09", "stderr": "0.028762349126466143"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-12-10", "model": "qwen3-235b-a22b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-235B-A22B-Thinking-2507", "source_row": {"Best score (across scorers)": "0.8666666666666667", "Country": "China", "Model version": "Qwen3-235B-A22B-Thinking-2507", "Organization": "Alibaba", "Release date": "2025-07-25", "Started at": "2025-12-10T18:03:27.753Z", "Training compute (FLOP)": "4.752e+24", "id": "XWpHpVPCaVjcwPpdpXuuGH", "mean_score": "0.8666666666666667", "stderr": "0.051247074319053836"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-12-11", "model": "qwen3-235b-a22b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8005050505050505, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-235B-A22B-Thinking-2507", "source_row": {"Best score (across scorers)": "0.8005050505050505", "Country": "China", "Model version": "Qwen3-235B-A22B-Thinking-2507", "Organization": "Alibaba", "Release date": "2025-07-25", "Started at": "2025-12-11T23:00:25.608Z", "Training compute (FLOP)": "4.752e+24", "id": "gu39ndv9QxWP3NRxSUJRaX", "mean_score": "0.8005050505050505", "stderr": "0.026000933515631548"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-22", "model": "qwen3-235b-a22b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7950335, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-235b-a22b-thinking-2507", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "rows": [{"accuracy": "0.7950335", "provider_name": "Alibaba", "total_questions": "792"}, {"accuracy": "0.8057394503311259", "provider_name": "DeepInfra", "total_questions": "604"}, {"accuracy": "0.71170025", "provider_name": "Novita", "total_questions": "792"}, {"accuracy": "0.753788", "provider_name": "Venice", "total_questions": "792"}], "rule": "first-party provider (1 of 4 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen3-235b-a22b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4044044044044044, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-235B-A22B-Thinking-2507", "source_row": {"Best score (across scorers)": "0.4044044044044044", "Country": "China", "Model version": "Qwen3-235B-A22B-Thinking-2507", "Organization": "Alibaba", "Release date": "2025-07-25", "Started at": "2026-08-27T19:36:29.000Z", "Training compute (FLOP)": "4.752e+24", "id": "4ACEC4x26HWRGiAepHGw43", "mean_score": "0.4044044044044044", "stderr": "0.015535255725079345"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-22", "model": "qwen3-235b-a22b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6266666666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-235b-a22b-thinking-2507", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-thinking-2507", "rows": [{"accuracy": "0.6266666666666667", "provider_name": "Alibaba", "total_questions": "150"}, {"accuracy": "0.5942761515151515", "provider_name": "Novita", "total_questions": "198"}, {"accuracy": "0.6377776666666667", "provider_name": "Venice", "total_questions": "150"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen-3-6-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-35b-a3b_none", "source_row": {"Best score (across scorers)": "0.22", "Country": "China", "Model version": "qwen3.6-35b-a3b_none", "Organization": "Alibaba", "Release date": "2026-04-14", "Started at": "2026-08-27T18:22:49.000Z", "id": "Ua5RK6JRxdyrHRa3hVFQwq", "mean_score": "0.22", "stderr": "0.041633319989322654"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-30", "model": "qwen-3-6-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.20350877192982456, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-35b-a3b_none", "source_row": {"Best score (across scorers)": "0.20350877192982456", "Country": "China", "Model version": "qwen3.6-35b-a3b_none", "Organization": "Alibaba", "Release date": "2026-04-14", "Started at": "2026-08-30T22:34:37.000Z", "id": "YRvn36dmMyDbS5e4XkF4Xi", "mean_score": "0.20350877192982456", "stderr": "0.02389037106974561"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen-3-6-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-35b-a3b", "source_row": {"Best score (across scorers)": "0.8666666666666667", "Country": "China", "Model version": "qwen3.6-35b-a3b", "Organization": "Alibaba", "Release date": "2026-04-14", "Started at": "2026-08-07T00:03:16.000Z", "id": "3n5KDcBaKqJh3WTg7qU2rs", "mean_score": "0.8666666666666667", "stderr": "0.051247074319053836"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen-3-6-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8484848484848485, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-35b-a3b_none", "source_row": {"Best score (across scorers)": "0.8484848484848485", "Country": "China", "Model version": "qwen3.6-35b-a3b_none", "Organization": "Alibaba", "Release date": "2026-04-14", "Started at": "2026-08-07T00:12:39.000Z", "id": "9kErpmgQ42wTvgJaFn478v", "mean_score": "0.8484848484848485", "stderr": "0.02554565042660364"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "qwen-3-6-35b-a3b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.8272847449686571, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.6-35b-a3b-20260415", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "rows": [{"accuracy": "0.8272847449686571", "provider_name": "AkashML", "total_questions": "3031"}, {"accuracy": "0.8481527067003793", "provider_name": "AtlasCloud", "total_questions": "2373"}, {"accuracy": "0.8308479888164026", "provider_name": "CoreWeave", "total_questions": "2146"}, {"accuracy": "0.6003243382761816", "provider_name": "Darkbloom", "total_questions": "2158"}, {"accuracy": "0.7000472153628653", "provider_name": "DeepInfra", "total_questions": "2122"}, {"accuracy": "0.785340410122164", "provider_name": "DekaLLM", "total_questions": "573"}, {"accuracy": "0.7465941264367815", "provider_name": "Io Net", "total_questions": "1566"}, {"accuracy": "0.8509335454545452", "provider_name": "Parasail", "total_questions": "2178"}, {"accuracy": "0.8261783333333332", "provider_name": "Phala", "total_questions": "2376"}, {"accuracy": "0.836139", "provider_name": "Reka", "total_questions": "594"}, {"accuracy": "0.836088", "provider_name": "SiliconFlow", "total_questions": "2178"}, {"accuracy": "0.8276515833333333", "provider_name": "Venice", "total_questions": "2376"}], "rule": "median over providers (12 of 12 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "qwen-3-6-35b-a3b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.744993393858478, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.6-35b-a3b-20260415", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.6-35b-a3b-20260415", "rows": [{"accuracy": "0.744993393858478", "provider_name": "AkashML", "total_questions": "749"}, {"accuracy": "0.7090909090909091", "provider_name": "AtlasCloud", "total_questions": "550"}, {"accuracy": "0.7528390961923848", "provider_name": "Darkbloom", "total_questions": "499"}, {"accuracy": "0.7520080321285141", "provider_name": "DeepInfra", "total_questions": "498"}, {"accuracy": "0.76046883919598", "provider_name": "DekaLLM", "total_questions": "199"}, {"accuracy": "0.4383965367088608", "provider_name": "Io Net", "total_questions": "395"}, {"accuracy": "0.752121090909091", "provider_name": "Parasail", "total_questions": "550"}, {"accuracy": "0.748484909090909", "provider_name": "Phala", "total_questions": "550"}, {"accuracy": "0.7444443333333334", "provider_name": "Reka", "total_questions": "150"}, {"accuracy": "0.2683333333333333", "provider_name": "Venice", "total_questions": "600"}], "rule": "median over providers (10 of 10 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_59K", "source_row": {"Best score (across scorers)": "0.1", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_59K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2026-10-13T13:46:15.000Z", "id": "RuQX7KA7owzqzf8Le9veeM", "mean_score": "0.1", "stderr": "0.03015113445777636"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-25", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_48K", "source_row": {"Best score (across scorers)": "0.17", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_48K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2026-07-25T07:06:00.000Z", "id": "4oSbA2RKFsfuAfVayAMAww", "mean_score": "0.17", "stderr": "0.03775251680686369"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.024390243902439025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_32K", "source_row": {"Best score (across scorers)": "0.024390243902439025", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_32K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2026-06-11T21:10:25.000Z", "id": "d3m9Xip8cof4s77z6LKkEh", "mean_score": "0.024390243902439025", "stderr": "0.024390243902439022"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23859649122807017, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_32K", "source_row": {"Best score (across scorers)": "0.23859649122807017", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_32K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2026-06-11T21:10:25.000Z", "id": "7oUkeLfwMK3MtkSNHX6QPu", "mean_score": "0.23859649122807017", "stderr": "0.02529183228411416"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-10-21", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9773413897280967, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_32K", "source_row": {"Best score (across scorers)": "0.9773413897280967", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_32K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2025-10-21T16:28:50.811Z", "id": "FqjAH25VNjgwQTbDJFfqAg", "mean_score": "0.9773413897280967", "stderr": "0.004091286174812221"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-28", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7777777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_59K", "source_row": {"Best score (across scorers)": "0.7777777777777778", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_59K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2025-10-28T10:21:02.293Z", "id": "k5PDL6MpCC6P5eLAhodJiS", "mean_score": "0.7777777777777778", "stderr": "0.06267511942419625"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-05", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7128099173553719, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929", "source_row": {"Best score (across scorers)": "0.7128099173553719", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2026-02-05T16:52:49.622Z", "id": "Lti64yMEsBWSzgEMFCSVhp", "mean_score": "0.7128099173553719", "stderr": "0.020587234937983246"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-10-28", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8232323232323232, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_59K", "source_row": {"Best score (across scorers)": "0.8232323232323232", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_59K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2025-10-28T13:46:37.314Z", "id": "XaGuWDxPtqVvyxqdMncKFj", "mean_score": "0.8232323232323232", "stderr": "0.027178752639044908"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8111110666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.5-sonnet-20250929", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "rows": [{"accuracy": "0.8159994814814814", "provider_name": "Amazon Bedrock", "total_questions": "5346"}, {"accuracy": "0.8111110666666667", "provider_name": "Anthropic", "total_questions": "2970"}, {"accuracy": "0.80780025", "provider_name": "Azure", "total_questions": "2376"}, {"accuracy": "0.8050504", "provider_name": "Claude Platform on AWS", "total_questions": "1980"}, {"accuracy": "0.8140431250000001", "provider_name": "Google", "total_questions": "4752"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.307, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-5-20250929_59K", "source_row": {"Best score (across scorers)": "0.307", "Country": "United States of America", "Model version": "claude-sonnet-4-5-20250929_59K", "Organization": "Anthropic", "Release date": "2025-09-29", "Started at": "2026-08-27T19:28:19.000Z", "id": "XQEGUQjpXupjdyKoGuSmum", "mean_score": "0.307", "stderr": "0.014593284892852741"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.7323999999999999, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Claude-Sonnet-4-5-20250929 (FC)", "source_row": {"Model": "Claude-Sonnet-4-5-20250929 (FC)", "Organization": "Anthropic", "Overall Acc": "73.24%", "Rank": "2"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-sonnet-4-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7528572857142857, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.5-sonnet-20250929", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-sonnet-20250929", "rows": [{"accuracy": "0.73413336", "provider_name": "Amazon Bedrock", "total_questions": "1250"}, {"accuracy": "0.7528572857142857", "provider_name": "Anthropic", "total_questions": "700"}, {"accuracy": "0.7260605454545455", "provider_name": "Azure", "total_questions": "550"}, {"accuracy": "0.742962888888889", "provider_name": "Claude Platform on AWS", "total_questions": "450"}, {"accuracy": "0.7369696818181818", "provider_name": "Google", "total_questions": "1100"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash-lite_low", "source_row": {"Best score (across scorers)": "0.21", "Country": "United States of America", "Model version": "gemini-3.5-flash-lite_low", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-10-13T13:25:11.000Z", "id": "d5QCJNq9uLtxjhiMhqXLmg", "mean_score": "0.21", "stderr": "0.040936018074033236"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-05", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.19, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash-lite_low", "source_row": {"Best score (across scorers)": "0.19", "Country": "United States of America", "Model version": "gemini-3.5-flash-lite_low", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-05T23:42:56.000Z", "id": "Mg2yb9Tp3yc2H3ApJX6JBK", "mean_score": "0.19", "stderr": "0.039427724440366255"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-02", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash-lite_high", "source_row": {"Best score (across scorers)": "0.0", "Country": "United States of America", "Model version": "gemini-3.5-flash-lite_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-02T01:20:17.000Z", "id": "juNVfKBcYTwaKnk9GRA2zL", "mean_score": "0.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-02", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2596491228070175, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash-lite_high", "source_row": {"Best score (across scorers)": "0.2596491228070175", "Country": "United States of America", "Model version": "gemini-3.5-flash-lite_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-02T01:20:17.000Z", "id": "8o2J9uQLKDCTiCFwYdkJRK", "mean_score": "0.2596491228070175", "stderr": "0.02601675082238553"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-06", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash-lite_high", "source_row": {"Best score (across scorers)": "0.7111111111111111", "Country": "United States of America", "Model version": "gemini-3.5-flash-lite_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-06T22:35:36.000Z", "id": "mqsc4vFuKtAHXxbYyX9WwK", "mean_score": "0.7111111111111111", "stderr": "0.0683294324254051"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8333333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.5-flash-lite_high", "source_row": {"Best score (across scorers)": "0.8333333333333334", "Country": "United States of America", "Model version": "gemini-3.5-flash-lite_high", "Organization": "Google DeepMind", "Release date": "2026-07-21", "Started at": "2026-08-06T22:35:04.000Z", "id": "chbh6XstiqtUa5W9zuyfjS", "mean_score": "0.8333333333333334", "stderr": "0.026552207828215258"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.8402775531250001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.5-flash-lite-20260721", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "rows": [{"accuracy": "0.8402775531250001", "provider_name": "Google", "total_questions": "1920"}, {"accuracy": "0.8413977096774194", "provider_name": "Google AI Studio", "total_questions": "744"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-3-5-flash-lite", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7349997500000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-3.5-flash-lite-20260721", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-3.5-flash-lite-20260721", "rows": [{"accuracy": "0.7593333", "provider_name": "Google", "total_questions": "500"}, {"accuracy": "0.7349997500000001", "provider_name": "Google AI Studio", "total_questions": "200"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro", "source_row": {"Best score (across scorers)": "0.0", "Country": "United States of America", "Model version": "gemini-2.5-pro", "Organization": "Google DeepMind", "Release date": "2025-06-17", "Started at": "2026-06-11T23:56:12.000Z", "id": "2z7kL9DpfvY44T9Wo99FYx", "mean_score": "0.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.24561403508771928, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro", "source_row": {"Best score (across scorers)": "0.24561403508771928", "Country": "United States of America", "Model version": "gemini-2.5-pro", "Organization": "Google DeepMind", "Release date": "2025-06-17", "Started at": "2026-06-11T23:56:12.000Z", "id": "FkywQ2KS4AkLPzjSPmhHcS", "mean_score": "0.24561403508771928", "stderr": "0.0255425481026507"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-11-16", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8472222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro", "source_row": {"Best score (across scorers)": "0.8472222222222222", "Country": "United States of America", "Model version": "gemini-2.5-pro", "Organization": "Google DeepMind", "Release date": "2025-06-17", "Started at": "2025-11-16T19:00:06.794Z", "id": "iJp7PgYZa3PiXAV8UWd7MC", "mean_score": "0.8416666666666667", "stderr": "0.048143196054711346"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-06-06", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.831, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gemini-2.5-pro-preview-06-05 (32k think)", "source_row": {"command": "aider --model gemini/gemini-2.5-pro-preview-06-05 --thinking-tokens 32k", "date": "2025-06-06", "dirname": "2025-06-06-16-36-21--gemini0605-32k-think-diff-fenced", "model": "gemini-2.5-pro-preview-06-05 (32k think)", "pass_rate_2": 83.1, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-13", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5755693581780539, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro", "source_row": {"Best score (across scorers)": "0.5755693581780539", "Country": "United States of America", "Model version": "gemini-2.5-pro", "Organization": "Google DeepMind", "Release date": "2025-06-17", "Started at": "2026-02-13T18:40:15.413Z", "id": "KL724uHFGbiXj4o5PHyLn2", "mean_score": "0.5755693581780539", "stderr": "0.02251275673889269"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-11-16", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8529040404040404, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro", "source_row": {"Best score (across scorers)": "0.8529040404040404", "Country": "United States of America", "Model version": "gemini-2.5-pro", "Organization": "Google DeepMind", "Release date": "2025-06-17", "Started at": "2025-11-16T19:00:08.124Z", "id": "cqUZ6gcq2HBnShFRWvKjGr", "mean_score": "0.8529040404040404", "stderr": "0.02117010842348584"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.806210222222222, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-2.5-pro", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-pro", "rows": [{"accuracy": "0.806210222222222", "provider_name": "Google", "total_questions": "1782"}, {"accuracy": "0.8223905000000001", "provider_name": "Google AI Studio", "total_questions": "396"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-2-5-pro-jun-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.5622223333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-2.5-pro", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-pro", "rows": [{"accuracy": "0.6060269955357143", "provider_name": "Google", "total_questions": "448"}, {"accuracy": "0.5622223333333333", "provider_name": "Google AI Studio", "total_questions": "150"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "minimax-m3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.13, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "MiniMax-M3", "source_row": {"Best score (across scorers)": "0.13", "Country": "China", "Model version": "MiniMax-M3", "Organization": "MiniMax", "Release date": "2026-06-01", "Started at": "2026-10-13T13:23:06.000Z", "id": "CtmxkTPbFRofQWTkEw5aSe", "mean_score": "0.13", "stderr": "0.03379976689896311"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "minimax-m3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "MiniMax-M3_none", "source_row": {"Best score (across scorers)": "0.08", "Country": "China", "Model version": "MiniMax-M3_none", "Organization": "MiniMax", "Release date": "2026-06-01", "Started at": "2026-08-27T18:09:34.000Z", "id": "jWHk7BTYHQRXk2rXCbnpcS", "mean_score": "0.08", "stderr": "0.027265992434429075"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-10", "model": "minimax-m3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "MiniMax-M3", "source_row": {"Best score (across scorers)": "0.7111111111111111", "Country": "China", "Model version": "MiniMax-M3", "Organization": "MiniMax", "Release date": "2026-06-01", "Started at": "2026-08-10T17:46:09.000Z", "id": "79ZP6iwoBaF5t9a5VRqPnw", "mean_score": "0.7111111111111111", "stderr": "0.0683294324254051"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-10", "model": "minimax-m3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9090909090909091, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "MiniMax-M3", "source_row": {"Best score (across scorers)": "0.9090909090909091", "Country": "China", "Model version": "MiniMax-M3", "Organization": "MiniMax", "Release date": "2026-06-01", "Started at": "2026-08-10T17:46:03.000Z", "id": "hDESmyGFrbZDomjff3rqzD", "mean_score": "0.9090909090909091", "stderr": "0.020482086775424225"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "minimax-m3", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.9151233333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m3-20260531", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m3-20260531", "rows": [{"accuracy": "0.8724903310344827", "provider_name": "AtlasCloud", "total_questions": "2175"}, {"accuracy": "0.9102327931985295", "provider_name": "CoreWeave", "total_questions": "2176"}, {"accuracy": "0.8959899189640768", "provider_name": "DeepInfra", "total_questions": "1197"}, {"accuracy": "0.9083225551330798", "provider_name": "GMICloud", "total_questions": "2367"}, {"accuracy": "0.8333332476370511", "provider_name": "Mara", "total_questions": "529"}, {"accuracy": "0.9151233333333333", "provider_name": "Minimax", "total_questions": "2376"}, {"accuracy": "0.8985086428571428", "provider_name": "ModelRun", "total_questions": "2772"}, {"accuracy": "0.8686612136752137", "provider_name": "Morph", "total_questions": "585"}, {"accuracy": "0.9067949999999999", "provider_name": "Novita", "total_questions": "2178"}, {"accuracy": "0.9000179188034189", "provider_name": "Parasail", "total_questions": "1872"}, {"accuracy": "0.8192149207920791", "provider_name": "SambaNova", "total_questions": "909"}, {"accuracy": "0.9069864999999999", "provider_name": "StreamLake", "total_questions": "2376"}, {"accuracy": "0.9073230999999999", "provider_name": "Together", "total_questions": "1980"}, {"accuracy": "0.9043450497466605", "provider_name": "Venice", "total_questions": "2171"}], "rule": "first-party provider (1 of 14 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "minimax-m3", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7388889166666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m3-20260531", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m3-20260531", "rows": [{"accuracy": "0.7331495477941177", "provider_name": "AtlasCloud", "total_questions": "544"}, {"accuracy": "0.7488888333333333", "provider_name": "CoreWeave", "total_questions": "300"}, {"accuracy": "0.7259886915254237", "provider_name": "DeepInfra", "total_questions": "295"}, {"accuracy": "0.7653333999999999", "provider_name": "GMICloud", "total_questions": "250"}, {"accuracy": "0.7468716348122866", "provider_name": "Mara", "total_questions": "293"}, {"accuracy": "0.7388889166666667", "provider_name": "Minimax", "total_questions": "600"}, {"accuracy": "0.7456411538461538", "provider_name": "ModelRun", "total_questions": "650"}, {"accuracy": "0.6977779999999999", "provider_name": "Morph", "total_questions": "150"}, {"accuracy": "0.7355556666666666", "provider_name": "Novita", "total_questions": "600"}, {"accuracy": "0.7444864177693761", "provider_name": "Parasail", "total_questions": "529"}, {"accuracy": "0.745", "provider_name": "SambaNova", "total_questions": "200"}, {"accuracy": "0.78", "provider_name": "StreamLake", "total_questions": "50"}, {"accuracy": "0.7681481111111111", "provider_name": "Together", "total_questions": "450"}, {"accuracy": "0.7248484545454545", "provider_name": "Venice", "total_questions": "550"}], "rule": "first-party provider (1 of 14 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-oss-120b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.02, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-oss-120b_medium", "source_row": {"Best score (across scorers)": "0.02", "Country": "United States of America", "Model version": "gpt-oss-120b_medium", "Organization": "OpenAI", "Release date": "2025-08-05", "Started at": "2026-08-27T17:57:43.000Z", "Training compute (FLOP)": "4.94e+24", "id": "58MQQiJ7vvgvP9e8LcG6eV", "mean_score": "0.02", "stderr": "0.014070529413628966"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-12-11", "model": "gpt-oss-120b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8888888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-oss-120b_high", "source_row": {"Best score (across scorers)": "0.8888888888888888", "Country": "United States of America", "Model version": "openai/gpt-oss-120b_high", "Organization": "OpenAI", "Release date": "2025-08-05", "Started at": "2025-12-11T22:37:41.702Z", "Training compute (FLOP)": "4.94e+24", "id": "3QjP5HPi8tKUzHtNJktfv4", "mean_score": "0.8888888888888888", "stderr": "0.04463343655613273"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-08-06", "model": "gpt-oss-120b", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.418, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-oss-120b (high)", "source_row": {"command": "aider --model openrouter/openai/gpt-oss-120b --reasoning-effort high", "date": "2025-08-06", "dirname": "2025-08-06-04-54-48--gpt-oss-120b-high-polyglot", "model": "gpt-oss-120b (high)", "pass_rate_2": 41.8, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-12-11", "model": "gpt-oss-120b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7575757575757576, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-oss-120b_high", "source_row": {"Best score (across scorers)": "0.7575757575757576", "Country": "United States of America", "Model version": "openai/gpt-oss-120b_high", "Organization": "OpenAI", "Release date": "2025-08-05", "Started at": "2025-12-11T22:37:44.523Z", "Training compute (FLOP)": "4.94e+24", "id": "LC5Rhog6wt2PT84CtJ483h", "mean_score": "0.7575757575757576", "stderr": "0.027437163443883767"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "gpt-oss-120b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7578456145038168, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-oss-120b", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-120b", "rows": [{"accuracy": "0.7664968651509212", "provider_name": "AkashML", "total_questions": "2551"}, {"accuracy": "0.7586279375000001", "provider_name": "Amazon Bedrock", "total_questions": "3168"}, {"accuracy": "0.7713499090909091", "provider_name": "BaseTen", "total_questions": "2178"}, {"accuracy": "0.5902780000000002", "provider_name": "Cerebras", "total_questions": "1584"}, {"accuracy": "0.7551755057034221", "provider_name": "CoreWeave", "total_questions": "1578"}, {"accuracy": "0.7620653333333334", "provider_name": "Crusoe", "total_questions": "594"}, {"accuracy": "0.7059358649303454", "provider_name": "DeepInfra", "total_questions": "3302"}, {"accuracy": "0.776515", "provider_name": "DekaLLM", "total_questions": "792"}, {"accuracy": "0.5947809999999999", "provider_name": "DigitalOcean", "total_questions": "1980"}, {"accuracy": "0.5994523122629583", "provider_name": "Google", "total_questions": "1582"}, {"accuracy": "0.75925925", "provider_name": "Groq", "total_questions": "1584"}, {"accuracy": "0.7037037142857143", "provider_name": "Mancer 2", "total_questions": "1386"}, {"accuracy": "0.7708977708978327", "provider_name": "Mara", "total_questions": "969"}, {"accuracy": "0.7630475000000001", "provider_name": "Near AI", "total_questions": "396"}, {"accuracy": "0.7558923750000001", "provider_name": "Nebius", "total_questions": "1584"}, {"accuracy": "0.7343086374045802", "provider_name": "Novita", "total_questions": "1572"}, {"accuracy": "0.7691922196969697", "provider_name": "Parasail", "total_questions": "1320"}, {"accuracy": "0.6417703596127247", "provider_name": "Phala", "total_questions": "1446"}, {"accuracy": "0.7664264643091028", "provider_name": "SambaNova", "total_questions": "1527"}, {"accuracy": "0.6777017260273973", "provider_name": "SiliconFlow", "total_questions": "1752"}, {"accuracy": "0.7578456145038168", "provider_name": "Together", "total_questions": "1572"}, {"accuracy": "0.7812311687388988", "provider_name": "Venice", "total_questions": "563"}], "rule": "median over providers (22 of 22 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "gpt-oss-120b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.633333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-oss-120b", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-120b", "rows": [{"accuracy": "0.652196480620155", "provider_name": "AkashML", "total_questions": "645"}, {"accuracy": "0.4358108783783784", "provider_name": "BaseTen", "total_questions": "296"}, {"accuracy": "0.619999875", "provider_name": "Cerebras", "total_questions": "400"}, {"accuracy": "0.5188173145161291", "provider_name": "CoreWeave", "total_questions": "248"}, {"accuracy": "0.6555553333333334", "provider_name": "Crusoe", "total_questions": "150"}, {"accuracy": "0.6409490933014353", "provider_name": "DeepInfra", "total_questions": "836"}, {"accuracy": "0.6791665", "provider_name": "DekaLLM", "total_questions": "200"}, {"accuracy": "0.6485186666666666", "provider_name": "Groq", "total_questions": "450"}, {"accuracy": "0.6006711006711409", "provider_name": "Mancer 2", "total_questions": "298"}, {"accuracy": "0.644208", "provider_name": "Mara", "total_questions": "141"}, {"accuracy": "0.6616665", "provider_name": "Near AI", "total_questions": "100"}, {"accuracy": "0.30916662500000003", "provider_name": "Nebius", "total_questions": "400"}, {"accuracy": "0.5687678137535817", "provider_name": "Novita", "total_questions": "349"}, {"accuracy": "0.6594443333333334", "provider_name": "Parasail", "total_questions": "300"}, {"accuracy": "0.6464647272727273", "provider_name": "Phala", "total_questions": "99"}, {"accuracy": "0.6239699483146067", "provider_name": "SambaNova", "total_questions": "445"}, {"accuracy": "0.5099998888888889", "provider_name": "Together", "total_questions": "450"}, {"accuracy": "0.633333", "provider_name": "Venice", "total_questions": "50"}], "rule": "median over providers (18 of 18 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.22", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-10-13T14:09:27.000Z", "id": "gQHCk4faPpTNupNZ3xmr4c", "mean_score": "0.22", "stderr": "0.041633319989322654"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.09, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_medium", "source_row": {"Best score (across scorers)": "0.09", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_medium", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-08-27T18:00:36.000Z", "id": "ZwQdGcba6Hez4ztMzmZP9j", "mean_score": "0.09", "stderr": "0.028762349126466143"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-12", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.024390243902439025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.024390243902439025", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-06-12T16:47:59.000Z", "id": "P5v3FUnVVTbaqaCr43deKi", "mean_score": "0.024390243902439025", "stderr": "0.024390243902439025"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-12", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.2", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-06-12T16:47:59.000Z", "id": "JH3PjDrbrQndEcc8fh5Qre", "mean_score": "0.2", "stderr": "0.023735633163877067"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-08-20", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.952416918429003, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_medium", "source_row": {"Best score (across scorers)": "0.952416918429003", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_medium", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-08-20T11:28:52.351Z", "id": "SSLfQCm7RvLhSv5aPKaLNk", "mean_score": "0.952416918429003", "stderr": "0.004627056893049539"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-31", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.8111111111111111", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-31T09:55:38.159Z", "id": "Y9YnaJQWpPFAPhe6jtKMhH", "mean_score": "0.8111111111111111", "stderr": "0.04626556218905063"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-10-30", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6944444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.6944444444444444", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2025-10-30T16:20:27.447Z", "id": "2pSDjMSGDh4HsAxkqUuNQP", "mean_score": "0.6944444444444444", "stderr": "0.027594508998251045"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6843435000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5-nano-2025-08-07", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "rows": [{"accuracy": "0.689815", "provider_name": "Azure", "total_questions": "792"}, {"accuracy": "0.6843435000000001", "provider_name": "OpenAI", "total_questions": "396"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.117, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-nano-2025-08-07_high", "source_row": {"Best score (across scorers)": "0.117", "Country": "United States of America", "Model version": "gpt-5-nano-2025-08-07_high", "Organization": "OpenAI", "Release date": "2025-08-07", "Started at": "2026-08-10T21:35:25.000Z", "id": "KgUCEfVCyRnY8B5kVChyiD", "mean_score": "0.117", "stderr": "0.010169287802713343"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5145000000000001, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "GPT-5-nano-2025-08-07 (FC)", "source_row": {"Model": "GPT-5-nano-2025-08-07 (FC)", "Organization": "OpenAI", "Overall Acc": "51.45%", "Rank": "24"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-nano", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.4511785656565656, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5-nano-2025-08-07", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5-nano-2025-08-07", "rows": [{"accuracy": "0.4608501677852349", "provider_name": "Azure", "total_questions": "298"}, {"accuracy": "0.4511785656565656", "provider_name": "OpenAI", "total_questions": "198"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.15, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.15", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2026-10-13T14:11:18.000Z", "id": "65oGRSHSEY6edS9mqbsCfE", "mean_score": "0.15", "stderr": "0.03588702812826367"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.07, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.07", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2026-08-27T19:08:51.000Z", "id": "JSmdEXGULNBJ7PfSoVoa7W", "mean_score": "0.07", "stderr": "0.02564323999762428"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.0", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2026-06-11T13:04:17.103Z", "id": "iR3Ut7aYEtz8dTYQFzq7rG", "mean_score": "0.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.18596491228070175, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.18596491228070175", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2026-06-11T13:04:17.668Z", "id": "mRJaJyBY2M2GGdbdMmT6iq", "mean_score": "0.18596491228070175", "stderr": "0.023087552563757593"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-13", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9648791540785498, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.9648791540785498", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2025-02-13T17:07:49.918Z", "id": "3VDhPT5WvzjJZkRc36Kuvi", "mean_score": "0.9648791540785498", "stderr": "0.004265263461984669"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-27", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7694444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.7694444444444445", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2025-02-27T14:29:45.288Z", "id": "Mjx6vYbgAb2DHXK8xcupX8", "mean_score": "0.7694444444444445", "stderr": "0.05239762076828802"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-01-31", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.5379999999999999, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "o3-mini (medium)", "source_row": {"command": "aider --model o3-mini", "date": "2025-01-31", "dirname": "2025-01-31-20-27-46--o3-mini-diff2", "model": "o3-mini (medium)", "pass_rate_2": 53.8, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-13", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7702020202020202, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.7702020202020202", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2025-02-13T17:07:50.336Z", "id": "MzsTvCEEfEpzn4WvxYdJkX", "mean_score": "0.7702020202020202", "stderr": "0.025855332698878485"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "o3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.153, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o3-mini-2025-01-31_high", "source_row": {"Best score (across scorers)": "0.153", "Country": "United States of America", "Model version": "o3-mini-2025-01-31_high", "Organization": "OpenAI", "Release date": "2025-01-31", "Started at": "2026-08-31T22:01:45.000Z", "id": "UCba4H3NNtUiWXMisWnh9g", "mean_score": "0.153", "stderr": "0.011389500459665504"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-4-5-20251001_59K", "source_row": {"Best score (across scorers)": "0.08", "Country": "United States of America", "Model version": "claude-haiku-4-5-20251001_59K", "Organization": "Anthropic", "Release date": "2025-10-15", "Started at": "2026-10-13T13:43:41.000Z", "id": "6bmTzBjpMySeHn65gc5Hu5", "mean_score": "0.08", "stderr": "0.027265992434429072"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-10-22", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9636087945413192, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-4-5-20251001_32K", "source_row": {"Best score (across scorers)": "0.9636087945413192", "Country": "United States of America", "Model version": "claude-haiku-4-5-20251001_32K", "Organization": "Anthropic", "Release date": "2025-10-15", "Started at": "2025-10-22T13:37:59.284Z", "id": "Qwci6JAG4DUWqA9eSHThDJ", "mean_score": "0.9636087945413192", "stderr": "0.00515811348923114"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-22", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6666666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-4-5-20251001_32K", "source_row": {"Best score (across scorers)": "0.6666666666666666", "Country": "United States of America", "Model version": "claude-haiku-4-5-20251001_32K", "Organization": "Anthropic", "Release date": "2025-10-15", "Started at": "2025-10-22T10:31:07.212Z", "id": "neZvEzLi6yeL9z5KjGef7H", "mean_score": "0.6666666666666666", "stderr": "0.07106690545187014"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-10-22", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7121212121212122, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-4-5-20251001_32K", "source_row": {"Best score (across scorers)": "0.7121212121212122", "Country": "United States of America", "Model version": "claude-haiku-4-5-20251001_32K", "Organization": "Anthropic", "Release date": "2025-10-15", "Started at": "2025-10-22T09:43:52.881Z", "id": "GEaAdb9UmoUFbEtBx7WzAR", "mean_score": "0.7121212121212122", "stderr": "0.03225883512300997"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7310605833333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.5-haiku-20251001", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "rows": [{"accuracy": "0.7312169142857143", "provider_name": "Amazon Bedrock", "total_questions": "6930"}, {"accuracy": "0.7310605833333333", "provider_name": "Anthropic", "total_questions": "2376"}, {"accuracy": "0.7271424333619948", "provider_name": "Azure", "total_questions": "2326"}, {"accuracy": "0.7215208055555558", "provider_name": "Google", "total_questions": "7128"}], "rule": "first-party provider (1 of 4 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-10", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.132, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-haiku-4-5-20251001", "source_row": {"Best score (across scorers)": "0.132", "Country": "United States of America", "Model version": "claude-haiku-4-5-20251001", "Organization": "Anthropic", "Release date": "2025-10-15", "Started at": "2026-08-10T21:26:22.000Z", "id": "J2kUEjR9r6Spd594qDNTTf", "mean_score": "0.132", "stderr": "0.010709373963527967"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.687, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Claude-Haiku-4-5-20251001 (FC)", "source_row": {"Model": "Claude-Haiku-4-5-20251001 (FC)", "Organization": "Anthropic", "Overall Acc": "68.70%", "Rank": "6"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "claude-haiku-4-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6657576363636364, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4.5-haiku-20251001", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4.5-haiku-20251001", "rows": [{"accuracy": "0.671666625", "provider_name": "Amazon Bedrock", "total_questions": "1600"}, {"accuracy": "0.6657576363636364", "provider_name": "Anthropic", "total_questions": "550"}, {"accuracy": "0.6515152727272728", "provider_name": "Azure", "total_questions": "550"}, {"accuracy": "0.659393909090909", "provider_name": "Google", "total_questions": "1650"}], "rule": "first-party provider (1 of 4 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-03-13", "model": "claude-3-7-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9116314199395771, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-7-sonnet-20250219_64K", "source_row": {"Best score (across scorers)": "0.9116314199395771", "Country": "United States of America", "Model version": "claude-3-7-sonnet-20250219_64K", "Organization": "Anthropic", "Release date": "2025-02-24", "Started at": "2025-03-13T21:33:10.936Z", "Training compute (FLOP)": "3.35e+25", "id": "epCyFNTf6vyUi5Dby7GeTq", "mean_score": "0.9116314199395771", "stderr": "0.007803306377082018"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-13", "model": "claude-3-7-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5777777777777777, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-7-sonnet-20250219_64K", "source_row": {"Best score (across scorers)": "0.5777777777777777", "Country": "United States of America", "Model version": "claude-3-7-sonnet-20250219_64K", "Organization": "Anthropic", "Release date": "2025-02-24", "Started at": "2025-03-13T21:20:00.444Z", "Training compute (FLOP)": "3.35e+25", "id": "QW3FUBn5qtAFEX4oQuzEAV", "mean_score": "0.5777777777777777", "stderr": "0.07446027270295805"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-02-24", "model": "claude-3-7-sonnet", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.649, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "claude-3-7-sonnet-20250219 (32k thinking tokens)", "source_row": {"command": "aider --model anthropic/claude-3-7-sonnet-20250219 --thinking-tokens 32k", "date": "2025-02-24", "dirname": "2025-02-24-21-47-23--sonnet37-diff-think-32k-64k", "model": "claude-3-7-sonnet-20250219 (32k thinking tokens)", "pass_rate_2": 64.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-04", "model": "claude-3-7-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.609504132231405, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-7-sonnet-20250219", "source_row": {"Best score (across scorers)": "0.609504132231405", "Country": "United States of America", "Model version": "claude-3-7-sonnet-20250219", "Organization": "Anthropic", "Release date": "2025-02-24", "Started at": "2026-02-04T18:05:21.852Z", "Training compute (FLOP)": "3.35e+25", "id": "cnM8U6moeqrp74wnXRj2Jj", "mean_score": "0.609504132231405", "stderr": "0.022198466971122266"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-05-26", "model": "claude-3-7-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7973484848484849, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-7-sonnet-20250219_64K", "source_row": {"Best score (across scorers)": "0.7973484848484849", "Country": "United States of America", "Model version": "claude-3-7-sonnet-20250219_64K", "Organization": "Anthropic", "Release date": "2025-02-24", "Started at": "2025-05-26T22:53:45.927Z", "Training compute (FLOP)": "3.35e+25", "id": "2t26VSNdsAE5pefQEMP25x", "mean_score": "0.7850378787878788", "stderr": "0.026506883123060064"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-05-22", "model": "claude-opus-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.850453172205438, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-20250514", "source_row": {"Best score (across scorers)": "0.850453172205438", "Country": "United States of America", "Model version": "claude-opus-4-20250514", "Organization": "Anthropic", "Release date": "2025-05-22", "Started at": "2025-05-22T17:39:30.135Z", "id": "YbrpPbWuYAXNqbSfU4so7f", "mean_score": "0.850453172205438", "stderr": "0.009804690367660946"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-05-28", "model": "claude-opus-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6444444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-20250514_27K", "source_row": {"Best score (across scorers)": "0.6444444444444445", "Country": "United States of America", "Model version": "claude-opus-4-20250514_27K", "Organization": "Anthropic", "Release date": "2025-05-22", "Started at": "2025-05-28T18:42:14.809Z", "id": "VrRccga4uRVuCjNp637MeY", "mean_score": "0.6444444444444445", "stderr": "0.0721639236343101"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-05-25", "model": "claude-opus-4", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.72, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "claude-opus-4-20250514 (32k thinking)", "source_row": {"command": "aider --model claude-opus-4-20250514", "date": "2025-05-25", "dirname": "2025-05-25-20-40-51--opus4-diff-exuser", "model": "claude-opus-4-20250514 (32k thinking)", "pass_rate_2": 72.0, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-06", "model": "claude-opus-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7066115702479339, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-20250514", "source_row": {"Best score (across scorers)": "0.7066115702479339", "Country": "United States of America", "Model version": "claude-opus-4-20250514", "Organization": "Anthropic", "Release date": "2025-05-22", "Started at": "2026-02-06T07:21:39.976Z", "id": "RbcvuVChvxK6SMQbPK8iRn", "mean_score": "0.7066115702479339", "stderr": "0.02071754535128056"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-05-22", "model": "claude-opus-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7626262626262627, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-20250514_16K", "source_row": {"Best score (across scorers)": "0.7626262626262627", "Country": "United States of America", "Model version": "claude-opus-4-20250514_16K", "Organization": "Anthropic", "Release date": "2025-05-22", "Started at": "2025-05-22T22:54:06.726Z", "id": "CtxFusPxCraL3EeRJu2c5k", "mean_score": "0.7626262626262627", "stderr": "0.03031371053819892"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-05-22", "model": "claude-sonnet-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8436555891238671, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-20250514", "source_row": {"Best score (across scorers)": "0.8436555891238671", "Country": "United States of America", "Model version": "claude-sonnet-4-20250514", "Organization": "Anthropic", "Release date": "2025-05-22", "Started at": "2025-05-22T17:39:31.972Z", "id": "dZLNhNKQVUgnB5ygAEUCcT", "mean_score": "0.8436555891238671", "stderr": "0.00998490304339695"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-05-22", "model": "claude-sonnet-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-20250514_32K", "source_row": {"Best score (across scorers)": "0.7111111111111111", "Country": "United States of America", "Model version": "claude-sonnet-4-20250514_32K", "Organization": "Anthropic", "Release date": "2025-05-22", "Started at": "2025-05-22T23:06:11.220Z", "id": "FZHzYjwAQexVykNQTnRtwT", "mean_score": "0.7111111111111111", "stderr": "0.06832943242540508"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-05-24", "model": "claude-sonnet-4", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.613, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "claude-sonnet-4-20250514 (32k thinking)", "source_row": {"command": "aider --model claude-sonnet-4-20250514", "date": "2025-05-24", "dirname": "2025-05-24-22-10-36--sonnet4-diff-exuser-think32k", "model": "claude-sonnet-4-20250514 (32k thinking)", "pass_rate_2": 61.3, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-05-26", "model": "claude-sonnet-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7918781725888325, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-sonnet-4-20250514_59K", "source_row": {"Best score (across scorers)": "0.7918781725888325", "Country": "United States of America", "Model version": "claude-sonnet-4-20250514_59K", "Organization": "Anthropic", "Release date": "2025-05-22", "Started at": "2025-05-26T18:38:03.090Z", "id": "J9wjcQqKJ7hCSYnnoa5E9i", "mean_score": "0.7781991873045173", "stderr": "0.02674821434321096"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-09-01", "model": "claude-sonnet-4", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7672559999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4-sonnet-20250522", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "rows": [{"accuracy": "0.7714368913043479", "provider_name": "Amazon Bedrock", "total_questions": "552"}, {"accuracy": "0.7672559999999999", "provider_name": "Google", "total_questions": "1188"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-09-01", "model": "claude-sonnet-4", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.64, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "anthropic/claude-4-sonnet-20250522", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "anthropic/claude-4-sonnet-20250522", "rows": [{"accuracy": "0.64", "provider_name": "Amazon Bedrock", "total_questions": "100"}, {"accuracy": "0.6622221666666667", "provider_name": "Google", "total_questions": "300"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "qwen3-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6694444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-32B", "source_row": {"Best score (across scorers)": "0.6694444444444444", "Country": "China", "Model version": "Qwen3-32B", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-30T22:44:49.000Z", "Training compute (FLOP)": "7.0848e+24", "id": "BAi4jidcT3F6reBkSXDJKv", "mean_score": "0.6694444444444444", "stderr": "0.06040106323530329"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-05-08", "model": "qwen3-32b", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.4, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Qwen3 32B", "source_row": {"command": "aider --model openrouter/qwen/qwen3-32b", "date": "2025-05-08", "dirname": "2025-05-08-03-20-24--qwen3-32b-default", "model": "Qwen3 32B", "pass_rate_2": 40.0, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "qwen3-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6571969696969697, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-32B", "source_row": {"Best score (across scorers)": "0.6571969696969697", "Country": "China", "Model version": "Qwen3-32B", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-28T17:33:22.000Z", "Training compute (FLOP)": "7.0848e+24", "id": "Ffoxqm7YsZr3FimCGsGAJv", "mean_score": "0.6571969696969697", "stderr": "0.027890331058144285"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-08", "model": "qwen3-32b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.5572391666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-32b-04-28", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-32b-04-28", "rows": [{"accuracy": "0.5572391666666667", "provider_name": "DeepInfra", "total_questions": "1188"}, {"accuracy": "0.638266870496592", "provider_name": "Groq", "total_questions": "1027"}, {"accuracy": "0.6050506", "provider_name": "Nebius", "total_questions": "990"}, {"accuracy": "0.5512932188317349", "provider_name": "Tenstorrent", "total_questions": "1147"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-32b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.48710000000000003, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-32B (FC)", "source_row": {"Model": "Qwen3-32B (FC)", "Organization": "Qwen", "Overall Acc": "48.71%", "Rank": "29"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-09", "model": "qwen3-32b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.4516665, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-32b-04-28", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-32b-04-28", "rows": [{"accuracy": "0.4516665", "provider_name": "DeepInfra", "total_questions": "300"}, {"accuracy": "0.4850950081300813", "provider_name": "Nebius", "total_questions": "246"}, {"accuracy": "0.5173608125000001", "provider_name": "SiliconFlow", "total_questions": "96"}, {"accuracy": "0.43434374747474747", "provider_name": "Tenstorrent", "total_questions": "99"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.01, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-mini-2025-04-14", "source_row": {"Best score (across scorers)": "0.01", "Country": "United States of America", "Model version": "gpt-4.1-mini-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-10-13T14:10:21.000Z", "id": "ZyRri9YZ8CNz6CrUWXFjBA", "mean_score": "0.01", "stderr": "0.009999999999999998"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.07, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-mini-2025-04-14", "source_row": {"Best score (across scorers)": "0.07", "Country": "United States of America", "Model version": "gpt-4.1-mini-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-08-27T17:58:41.000Z", "id": "giKk2ut3JVFVQUvbykxvSu", "mean_score": "0.07", "stderr": "0.02564323999762428"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-mini-2025-04-14", "source_row": {"Best score (across scorers)": "0.06666666666666667", "Country": "United States of America", "Model version": "gpt-4.1-mini-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-08-27T19:59:44.000Z", "id": "6EQrp2MY4QMYLfTxpioYTe", "mean_score": "0.06666666666666667", "stderr": "0.014801767859562859"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-14", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8729229607250756, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-mini-2025-04-14", "source_row": {"Best score (across scorers)": "0.8729229607250756", "Country": "United States of America", "Model version": "gpt-4.1-mini-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T22:50:03.233Z", "id": "6ybVweWdTfiCuirnS2BKXD", "mean_score": "0.8729229607250756", "stderr": "0.007298484758774334"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-14", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.44722222222222224, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-mini-2025-04-14", "source_row": {"Best score (across scorers)": "0.44722222222222224", "Country": "United States of America", "Model version": "gpt-4.1-mini-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T22:49:56.953Z", "id": "cXbbdQDW2jeYassmRuZYpU", "mean_score": "0.44722222222222224", "stderr": "0.06148017573965347"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-14", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.324, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-4.1-mini", "source_row": {"command": "aider --model gpt-4.1-mini", "date": "2025-04-14", "dirname": "2025-04-14-21-27-53--gpt41mini-diff", "model": "gpt-4.1-mini", "pass_rate_2": 32.4, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-14", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6584595959595959, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-mini-2025-04-14", "source_row": {"Best score (across scorers)": "0.6584595959595959", "Country": "United States of America", "Model version": "gpt-4.1-mini-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T22:49:57.790Z", "id": "dXHddeGRX7JRVa5UNfad5C", "mean_score": "0.6584595959595959", "stderr": "0.027051500965295915"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6473065000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4.1-mini-2025-04-14", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "rows": [{"accuracy": "0.6405723333333334", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.6473065000000001", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.127, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-mini-2025-04-14", "source_row": {"Best score (across scorers)": "0.127", "Country": "United States of America", "Model version": "gpt-4.1-mini-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-08-31T22:01:51.000Z", "id": "gYV4CJTjSd7VonfzMmnwcw", "mean_score": "0.127", "stderr": "0.010534798620855644"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5045000000000001, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "GPT-4.1-mini-2025-04-14 (FC)", "source_row": {"Model": "GPT-4.1-mini-2025-04-14 (FC)", "Organization": "OpenAI", "Overall Acc": "50.45%", "Rank": "27"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-4-1-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.44333325, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4.1-mini-2025-04-14", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-mini-2025-04-14", "rows": [{"accuracy": "0.40777783333333334", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.44333325", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.03, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-2025-04-14", "source_row": {"Best score (across scorers)": "0.03", "Country": "United States of America", "Model version": "gpt-4.1-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-10-13T14:12:01.000Z", "id": "TzdbTbqAFB73Tfet8GFzQQ", "mean_score": "0.03", "stderr": "0.01714466079977653"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.05964912280701754, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-2025-04-14", "source_row": {"Best score (across scorers)": "0.05964912280701754", "Country": "United States of America", "Model version": "gpt-4.1-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-08-27T17:54:56.000Z", "id": "kCG2nzspryMy7LBwWPqwXy", "mean_score": "0.05964912280701754", "stderr": "0.014053607746337793"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-14", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8300604229607251, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-2025-04-14", "source_row": {"Best score (across scorers)": "0.8300604229607251", "Country": "United States of America", "Model version": "gpt-4.1-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T22:22:31.727Z", "id": "YyJXPVXJjN5EnLfYjfCTrQ", "mean_score": "0.8300604229607251", "stderr": "0.008490146044318322"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-14", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.38333333333333336, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-2025-04-14", "source_row": {"Best score (across scorers)": "0.38333333333333336", "Country": "United States of America", "Model version": "gpt-4.1-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T18:34:29.241Z", "id": "VAQtARXUPTpdxGKMekUSbX", "mean_score": "0.38333333333333336", "stderr": "0.06324056203789373"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-14", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.524, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-4.1", "source_row": {"command": "aider --model gpt-4.1", "date": "2025-04-14", "dirname": "2025-04-14-21-05-54--gpt41-diff-exuser", "model": "gpt-4.1", "pass_rate_2": 52.4, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-08", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48541666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-2025-04-14", "source_row": {"Best score (across scorers)": "0.48541666666666666", "Country": "United States of America", "Model version": "gpt-4.1-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-02-08T17:41:23.879Z", "id": "SheUt9cNpkSevy4r6Gsr3u", "mean_score": "0.48541666666666666", "stderr": "0.022835863746433965"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-14", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6691919191919192, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-2025-04-14", "source_row": {"Best score (across scorers)": "0.6691919191919192", "Country": "United States of America", "Model version": "gpt-4.1-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T18:34:33.975Z", "id": "Gj8pd5gU9r4LqwDK8XZ4mL", "mean_score": "0.6691919191919192", "stderr": "0.02821440093855908"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6607744999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4.1-2025-04-14", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-2025-04-14", "rows": [{"accuracy": "0.643659", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.6607744999999999", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.311, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-2025-04-14", "source_row": {"Best score (across scorers)": "0.311", "Country": "United States of America", "Model version": "gpt-4.1-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-08-31T22:01:48.000Z", "id": "avdYKfKUuY6EvfVdxftBE8", "mean_score": "0.311", "stderr": "0.014645596385722692"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5396, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "GPT-4.1-2025-04-14 (FC)", "source_row": {"Model": "GPT-4.1-2025-04-14 (FC)", "Organization": "OpenAI", "Overall Acc": "53.96%", "Rank": "20"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-4-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.5083332500000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4.1-2025-04-14", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-2025-04-14", "rows": [{"accuracy": "0.5221089591836734", "provider_name": "Azure", "total_questions": "196"}, {"accuracy": "0.46", "provider_name": "FakeProvider", "total_questions": "100"}, {"accuracy": "0.5083332500000001", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-04-08", "model": "llama-4-maverick", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7301737160120846, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-4-Maverick-17B-128E-Instruct-FP8", "source_row": {"Best score (across scorers)": "0.7301737160120846", "Country": "United States of America", "Model version": "Llama-4-Maverick-17B-128E-Instruct-FP8", "Organization": "Meta AI", "Release date": "2025-04-05", "Started at": "2025-04-08T09:46:43.408Z", "Training compute (FLOP)": "2.244000000001e+24", "id": "HkjCosgMFq5QXB2CyTyU7r", "mean_score": "0.7301737160120846", "stderr": "0.01083573699636281"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-08", "model": "llama-4-maverick", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.20555555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-4-Maverick-17B-128E-Instruct-FP8", "source_row": {"Best score (across scorers)": "0.20555555555555555", "Country": "United States of America", "Model version": "Llama-4-Maverick-17B-128E-Instruct-FP8", "Organization": "Meta AI", "Release date": "2025-04-05", "Started at": "2025-04-08T09:58:55.652Z", "Training compute (FLOP)": "2.244000000001e+24", "id": "GJEpPThqUxXJBSbvkuXVnw", "mean_score": "0.20555555555555555", "stderr": "0.04948528559541873"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-06", "model": "llama-4-maverick", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.156, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Llama 4 Maverick", "source_row": {"command": "aider --model nvidia_nim/meta/llama-4-maverick-17b-128e-instruct", "date": "2025-04-06", "dirname": "2025-04-06-08-39-52--llama-4-maverick-17b-128e-instruct-polyglot-whole", "model": "Llama 4 Maverick", "pass_rate_2": 15.6, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-08", "model": "llama-4-maverick", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6698232323232324, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-4-Maverick-17B-128E-Instruct-FP8", "source_row": {"Best score (across scorers)": "0.6698232323232324", "Country": "United States of America", "Model version": "Llama-4-Maverick-17B-128E-Instruct-FP8", "Organization": "Meta AI", "Release date": "2025-04-05", "Started at": "2025-04-08T08:34:49.725Z", "Training compute (FLOP)": "2.244000000001e+24", "id": "LyHy99ubGCqBhoaksQjkiF", "mean_score": "0.6698232323232324", "stderr": "0.028244823529728173"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-29", "model": "llama-4-maverick", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.6478116, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta-llama/llama-4-maverick-17b-128e-instruct", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "rows": [{"accuracy": "0.6478116", "provider_name": "DeepInfra", "total_questions": "990"}, {"accuracy": "0.6661054999999999", "provider_name": "DigitalOcean", "total_questions": "1188"}, {"accuracy": "0.6454544", "provider_name": "Novita", "total_questions": "990"}, {"accuracy": "0.6505050000000001", "provider_name": "Parasail", "total_questions": "990"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "llama-4-maverick", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.3729, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", "source_row": {"Model": "Llama-4-Maverick-17B-128E-Instruct-FP8 (FC)", "Organization": "Meta", "Overall Acc": "37.29%", "Rank": "50"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-29", "model": "llama-4-maverick", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.43089426016260163, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta-llama/llama-4-maverick-17b-128e-instruct", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-4-maverick-17b-128e-instruct", "rows": [{"accuracy": "0.4391890202702703", "provider_name": "DigitalOcean", "total_questions": "296"}, {"accuracy": "0.43089426016260163", "provider_name": "Parasail", "total_questions": "246"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-03-13", "model": "gemma-3-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.740370090634441, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-27b-it", "source_row": {"Best score (across scorers)": "0.740370090634441", "Country": "United States of America", "Model version": "gemma-3-27b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2025-03-13T05:52:35.397Z", "Training compute (FLOP)": "2.268e+24", "id": "amrtN7UjAtuyEmv5DaGshP", "mean_score": "0.740370090634441", "stderr": "0.011195857525727288"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-27", "model": "gemma-3-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.225, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-27b-it", "source_row": {"Best score (across scorers)": "0.225", "Country": "United States of America", "Model version": "gemma-3-27b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-27T14:17:59.000Z", "Training compute (FLOP)": "2.268e+24", "id": "DhJ248BX3ZPBHXvkRG4Qsz", "mean_score": "0.225", "stderr": "0.052794388667238824"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-03-15", "model": "gemma-3-27b", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.049, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gemma-3-27b-it", "source_row": {"command": "aider --model openrouter/google/gemma-3-27b-it", "date": "2025-03-15", "dirname": "2025-03-15-01-21-24--gemma3-27b-or", "model": "gemma-3-27b-it", "pass_rate_2": 4.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-27", "model": "gemma-3-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4772727272727273, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-27b-it", "source_row": {"Best score (across scorers)": "0.4772727272727273", "Country": "United States of America", "Model version": "gemma-3-27b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-27T14:17:43.000Z", "Training compute (FLOP)": "2.268e+24", "id": "jdFH2zZdssYKDVJQPKXvNB", "mean_score": "0.4772727272727273", "stderr": "0.026982120891168943"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gemma-3-27b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.29469999999999996, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Gemma-3-27b-it (Prompt)", "source_row": {"Model": "Gemma-3-27b-it (Prompt)", "Organization": "Google", "Overall Acc": "29.47%", "Rank": "69"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-nano-2025-04-14", "source_row": {"Best score (across scorers)": "0.0", "Country": "United States of America", "Model version": "gpt-4.1-nano-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-10-13T14:11:28.000Z", "id": "FLJ6bNrtaJQGCU2dP3CvWH", "mean_score": "0.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-14", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6999622356495468, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-nano-2025-04-14", "source_row": {"Best score (across scorers)": "0.6999622356495468", "Country": "United States of America", "Model version": "gpt-4.1-nano-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T22:49:56.785Z", "id": "K4TfMXsaPd4Z46zFmiV4Rf", "mean_score": "0.6999622356495468", "stderr": "0.010231591350302844"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-14", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.28888888888888886, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-nano-2025-04-14", "source_row": {"Best score (across scorers)": "0.28888888888888886", "Country": "United States of America", "Model version": "gpt-4.1-nano-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T22:50:00.937Z", "id": "3qLHwibDhgtWEbKGwxUYVa", "mean_score": "0.28888888888888886", "stderr": "0.05970019560922746"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-14", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.08900000000000001, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-4.1-nano", "source_row": {"command": "aider --model gpt-4.1-nano", "date": "2025-04-14", "dirname": "2025-04-14-22-46-01--gpt41nano-diff", "model": "gpt-4.1-nano", "pass_rate_2": 8.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-14", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48926767676767674, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-nano-2025-04-14", "source_row": {"Best score (across scorers)": "0.48926767676767674", "Country": "United States of America", "Model version": "gpt-4.1-nano-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2025-04-14T22:49:56.708Z", "id": "azWfeFPP6zSocuQX3JYBU5", "mean_score": "0.48926767676767674", "stderr": "0.024849734950435368"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.4966332500000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4.1-nano-2025-04-14", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "rows": [{"accuracy": "0.49764320000000006", "provider_name": "Azure", "total_questions": "990"}, {"accuracy": "0.4966332500000001", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.1-nano-2025-04-14", "source_row": {"Best score (across scorers)": "0.06", "Country": "United States of America", "Model version": "gpt-4.1-nano-2025-04-14", "Organization": "OpenAI", "Release date": "2025-04-14", "Started at": "2026-08-31T22:01:58.000Z", "id": "A4JEeJR5LQfLR3Hb8z7kL2", "mean_score": "0.06", "stderr": "0.007513751157474971"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.33049999999999996, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "GPT-4.1-nano-2025-04-14 (FC)", "source_row": {"Model": "GPT-4.1-nano-2025-04-14 (FC)", "Organization": "OpenAI", "Overall Acc": "33.05%", "Rank": "58"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-4-1-nano", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.11833325, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4.1-nano-2025-04-14", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4.1-nano-2025-04-14", "rows": [{"accuracy": "0.13333333333333333", "provider_name": "Azure", "total_questions": "300"}, {"accuracy": "0.11833325", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-08-06", "source_row": {"Best score (across scorers)": "0.06", "Country": "United States of America", "Model version": "gpt-4o-2024-08-06", "Organization": "OpenAI", "Release date": "2024-08-06", "Started at": "2026-10-13T14:12:01.000Z", "id": "Cjt9Xa7iPVBf9dN5cfJ88q", "mean_score": "0.06", "stderr": "0.023868325657594204"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0035087719298245615, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-08-06", "source_row": {"Best score (across scorers)": "0.0035087719298245615", "Country": "United States of America", "Model version": "gpt-4o-2024-08-06", "Organization": "OpenAI", "Release date": "2024-08-06", "Started at": "2026-08-27T17:54:29.000Z", "id": "ikS95CXY2foitnUPDKwNJn", "mean_score": "0.0035087719298245615", "stderr": "0.0035087719298245615"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5327605740181269, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-08-06", "source_row": {"Best score (across scorers)": "0.5327605740181269", "Country": "United States of America", "Model version": "gpt-4o-2024-08-06", "Organization": "OpenAI", "Release date": "2024-08-06", "Started at": "2025-01-27T00:00:00.000Z", "id": "ZQgz8AGm4KHin9MFupLu58", "mean_score": "0.5327605740181269", "stderr": "0.01122129021441016"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06388888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-08-06", "source_row": {"Best score (across scorers)": "0.06388888888888888", "Country": "United States of America", "Model version": "gpt-4o-2024-08-06", "Organization": "OpenAI", "Release date": "2024-08-06", "Started at": "2025-02-25T14:05:08.205Z", "id": "mBUKqiVQsqXd3qmdtKP5UP", "mean_score": "0.06388888888888888", "stderr": "0.025630380139261548"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-30", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.231, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-4o-2024-08-06", "source_row": {"command": "aider --model gpt-4o-2024-08-06", "date": "2024-12-30", "dirname": "2024-12-30-20-44-54--gpt4o-ex-as-sys-clean-prompt", "model": "gpt-4o-2024-08-06", "pass_rate_2": 23.1, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.49210858585858586, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-08-06", "source_row": {"Best score (across scorers)": "0.49210858585858586", "Country": "United States of America", "Model version": "gpt-4o-2024-08-06", "Organization": "OpenAI", "Release date": "2024-08-06", "Started at": "2025-01-27T00:00:00.000Z", "id": "fairJpL9VekDBKxYHG9ZoG", "mean_score": "0.49210858585858586", "stderr": "0.025999270277271707"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.5280586666666666, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4o-2024-08-06", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-08-06", "rows": [{"accuracy": "0.5151519999999999", "provider_name": "Azure", "total_questions": "396"}, {"accuracy": "0.5280586666666666", "provider_name": "OpenAI", "total_questions": "594"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-08-06", "source_row": {"Best score (across scorers)": "0.26", "Country": "United States of America", "Model version": "gpt-4o-2024-08-06", "Organization": "OpenAI", "Release date": "2024-08-06", "Started at": "2026-08-31T22:02:02.000Z", "id": "bDFqfWxMG7gLDq4Mz8PRTP", "mean_score": "0.26", "stderr": "0.01387777332977422"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-4o-aug-2024", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.43, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4o-2024-08-06", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-08-06", "rows": [{"accuracy": "0.43", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-02-05", "model": "gpt-4o-nov-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.49773413897280966, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-11-20", "source_row": {"Best score (across scorers)": "0.49773413897280966", "Country": "United States of America", "Model version": "gpt-4o-2024-11-20", "Organization": "OpenAI", "Release date": "2024-11-20", "Started at": "2025-02-05T08:31:16.384Z", "id": "D3G2BKGnh6YSvw4bEYSdiY", "mean_score": "0.49773413897280966", "stderr": "0.011102819618096572"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gpt-4o-nov-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0625, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-11-20", "source_row": {"Best score (across scorers)": "0.0625", "Country": "United States of America", "Model version": "gpt-4o-2024-11-20", "Organization": "OpenAI", "Release date": "2024-11-20", "Started at": "2025-02-25T13:10:02.200Z", "id": "buNVL2TWcdVgxAcMfVE8xJ", "mean_score": "0.0625", "stderr": "0.024165360466267546"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-30", "model": "gpt-4o-nov-2024", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.182, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-4o-2024-11-20", "source_row": {"command": "aider --model gpt-4o-2024-11-20", "date": "2024-12-30", "dirname": "2024-12-30-20-57-12--gpt-4o-2024-11-20-ex-as-sys", "model": "gpt-4o-2024-11-20", "pass_rate_2": 18.2, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-11", "model": "gpt-4o-nov-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.30991735537190085, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-11-20", "source_row": {"Best score (across scorers)": "0.30991735537190085", "Country": "United States of America", "Model version": "gpt-4o-2024-11-20", "Organization": "OpenAI", "Release date": "2024-11-20", "Started at": "2026-02-11T15:05:15.089Z", "id": "PncXWJm7ZCY3qv982pKoRy", "mean_score": "0.30991735537190085", "stderr": "0.021042625979133336"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-05", "model": "gpt-4o-nov-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4788510101010101, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-11-20", "source_row": {"Best score (across scorers)": "0.4788510101010101", "Country": "United States of America", "Model version": "gpt-4o-2024-11-20", "Organization": "OpenAI", "Release date": "2024-11-20", "Started at": "2025-02-05T08:31:15.914Z", "id": "gULKGqAe9nG9syxmYMtkhi", "mean_score": "0.4788510101010101", "stderr": "0.026336972178726107"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.03, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"Best score (across scorers)": "0.03", "Country": "United States of America", "Model version": "gpt-4o-mini-2024-07-18", "Organization": "OpenAI", "Release date": "2024-07-18", "Started at": "2026-10-13T14:12:48.000Z", "id": "QGrgPmo24qW4h9SAvELP3Y", "mean_score": "0.03", "stderr": "0.01714466079977653"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"Best score (across scorers)": "0.12", "Country": "United States of America", "Model version": "gpt-4o-mini-2024-07-18", "Organization": "OpenAI", "Release date": "2024-07-18", "Started at": "2026-08-27T17:56:46.000Z", "id": "mbviVfQ7Bh7wHSCDDmrt7u", "mean_score": "0.12", "stderr": "0.03265986323710904"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.007017543859649123, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"Best score (across scorers)": "0.007017543859649123", "Country": "United States of America", "Model version": "gpt-4o-mini-2024-07-18", "Organization": "OpenAI", "Release date": "2024-07-18", "Started at": "2026-08-27T19:51:52.000Z", "id": "6XhtKr59amKko65T7RUB2a", "mean_score": "0.007017543859649123", "stderr": "0.004953408961963604"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5263406344410876, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"Best score (across scorers)": "0.5263406344410876", "Country": "United States of America", "Model version": "gpt-4o-mini-2024-07-18", "Organization": "OpenAI", "Release date": "2024-07-18", "Started at": "2025-01-27T00:00:00.000Z", "id": "5iyKgWVhAmNwrA5rNL7MbW", "mean_score": "0.5263406344410876", "stderr": "0.011275275696871733"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-07-30", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06944444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"Best score (across scorers)": "0.06944444444444445", "Country": "United States of America", "Model version": "gpt-4o-mini-2024-07-18", "Organization": "OpenAI", "Release date": "2024-07-18", "Started at": "2025-07-30T15:47:06.669Z", "id": "MPJkYPEMsQrowqtecK3mmz", "mean_score": "0.06944444444444445", "stderr": "0.0273963714244758"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-21", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.036000000000000004, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"command": "aider --model gpt-4o-mini-2024-07-18", "date": "2024-12-21", "dirname": "2024-12-21-18-41-18--polyglot-gpt-4o-mini", "model": "gpt-4o-mini-2024-07-18", "pass_rate_2": 3.6, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37720959595959597, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"Best score (across scorers)": "0.37720959595959597", "Country": "United States of America", "Model version": "gpt-4o-mini-2024-07-18", "Organization": "OpenAI", "Release date": "2024-07-18", "Started at": "2025-01-27T00:00:00.000Z", "id": "L9yfwZC3snsKZshBXTaXjc", "mean_score": "0.37720959595959597", "stderr": "0.024068925424620136"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-02", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.4305555, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4o-mini", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-mini", "rows": [{"accuracy": "0.4315373333333334", "provider_name": "Azure", "total_questions": "1188"}, {"accuracy": "0.4305555", "provider_name": "OpenAI", "total_questions": "792"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.083, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-mini-2024-07-18", "source_row": {"Best score (across scorers)": "0.083", "Country": "United States of America", "Model version": "gpt-4o-mini-2024-07-18", "Organization": "OpenAI", "Release date": "2024-07-18", "Started at": "2026-08-31T22:02:07.000Z", "id": "DLr586bZUHpghAf8m3GRSm", "mean_score": "0.083", "stderr": "0.008728527206074758"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-4o-mini", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.28000025, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4o-mini", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-mini", "rows": [{"accuracy": "0.28000025", "provider_name": "OpenAI", "total_questions": "200"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "llama-3-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.41597432024169184, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.3-70B-Instruct", "source_row": {"Best score (across scorers)": "0.41597432024169184", "Country": "United States of America", "Model version": "Llama-3.3-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-12-06", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "6.8649768e+24", "id": "DeaDxFfiQtCTdnZPH7Awaf", "mean_score": "0.41597432024169184", "stderr": "0.011816670264886683"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "llama-3-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.05138888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.3-70B-Instruct", "source_row": {"Best score (across scorers)": "0.05138888888888889", "Country": "United States of America", "Model version": "Llama-3.3-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-12-06", "Started at": "2025-02-25T14:05:10.812Z", "Training compute (FLOP)": "6.8649768e+24", "id": "ZjsXQpZ7SsejXDo52aetZZ", "mean_score": "0.05138888888888889", "stderr": "0.023611111111111107"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "llama-3-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4744318181818182, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.3-70B-Instruct", "source_row": {"Best score (across scorers)": "0.4744318181818182", "Country": "United States of America", "Model version": "Llama-3.3-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-12-06", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "6.8649768e+24", "id": "TxXS78Wg2pSCDbpmeJuzQn", "mean_score": "0.4744318181818182", "stderr": "0.028523027562061026"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "llama-3-3-70b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.4753785, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta-llama/llama-3.3-70b-instruct", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "rows": [{"accuracy": "0.48224677424483314", "provider_name": "AkashML", "total_questions": "2516"}, {"accuracy": "0.40225624573378843", "provider_name": "Cloudflare", "total_questions": "1758"}, {"accuracy": "0.48042899999999994", "provider_name": "CoreWeave", "total_questions": "1584"}, {"accuracy": "0.46742483824768327", "provider_name": "Crusoe", "total_questions": "1187"}, {"accuracy": "0.4634305555555556", "provider_name": "DeepInfra", "total_questions": "1782"}, {"accuracy": "0.49831644444444445", "provider_name": "Groq", "total_questions": "1782"}, {"accuracy": "0.47777759999999997", "provider_name": "Nebius", "total_questions": "990"}, {"accuracy": "0.4717298835443038", "provider_name": "Novita", "total_questions": "1580"}, {"accuracy": "0.4753785", "provider_name": "Parasail", "total_questions": "1584"}, {"accuracy": "0.469697", "provider_name": "PrimeIntellect", "total_questions": "198"}, {"accuracy": "0.48550299999999996", "provider_name": "SambaNova", "total_questions": "1782"}, {"accuracy": "0.4758696666666667", "provider_name": "Together", "total_questions": "1782"}], "rule": "median over providers (12 of 12 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "llama-3-3-70b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.319, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Llama-3.3-70B-Instruct (FC)", "source_row": {"Model": "Llama-3.3-70B-Instruct (FC)", "Organization": "Meta", "Overall Acc": "31.90%", "Rank": "62"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-04", "model": "llama-3-3-70b", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.313333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta-llama/llama-3.3-70b-instruct", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "meta-llama/llama-3.3-70b-instruct", "rows": [{"accuracy": "0.37148306580829754", "provider_name": "AkashML", "run_count": "14", "total_questions": "699"}, {"accuracy": "0.35925922222222223", "provider_name": "CoreWeave", "run_count": "9", "total_questions": "450"}, {"accuracy": "0.3093607534246575", "provider_name": "DeepInfra", "run_count": "6", "total_questions": "292"}, {"accuracy": "0.3877778333333333", "provider_name": "Nebius", "run_count": "6", "total_questions": "300"}, {"accuracy": "0.313333", "provider_name": "PrimeIntellect", "run_count": "1", "total_questions": "50"}, {"accuracy": "0.29925922222222223", "provider_name": "Together", "run_count": "9", "total_questions": "450"}], "rule": "median over providers (6 of 6 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "gpt-5-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.62, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pro_xhigh", "source_row": {"Best score (across scorers)": "0.62", "Country": "United States of America", "Model version": "gpt-5.5-pro_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-10-14T09:56:16.000Z", "id": "5u4XrDnQSRnuXSK6oXKAth", "mean_score": "0.62", "stderr": "0.04878317312145634"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-12", "model": "gpt-5-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7804878048780488, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pro_xhigh", "source_row": {"Best score (across scorers)": "0.7804878048780488", "Country": "United States of America", "Model version": "gpt-5.5-pro_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-06-12T21:33:12.000Z", "id": "SFKXCPUu73mwxMoUotiZM2", "mean_score": "0.7804878048780488", "stderr": "0.06544589202438408"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-12", "model": "gpt-5-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8771929824561403, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pro_xhigh", "source_row": {"Best score (across scorers)": "0.8771929824561403", "Country": "United States of America", "Model version": "gpt-5.5-pro_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-06-12T21:33:12.000Z", "id": "LSKDetgzyVojru45w7RJhf", "mean_score": "0.8771929824561403", "stderr": "0.019476010341530077"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-04-24", "model": "gpt-5-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pro-pre-release_xhigh", "source_row": {"Best score (across scorers)": "1.0", "Country": "United States of America", "Model version": "gpt-5.5-pro-pre-release_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-04-24T23:38:07.000Z", "id": "YxbseUDNgFUjokfYXui62r", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-04-24", "model": "gpt-5-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9392135642135643, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-pro-pre-release_xhigh", "source_row": {"Best score (across scorers)": "0.9392135642135643", "Country": "United States of America", "Model version": "gpt-5.5-pro-pre-release_xhigh", "Organization": "OpenAI", "Release date": "2026-04-23", "Started at": "2026-04-24T16:01:38.000Z", "id": "cBmhJzrC3SkfZ7uVgGVYZX", "mean_score": "0.9392135642135643", "stderr": "0.015832823810775415"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-12", "model": "ai-co-mathematician", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.756, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gdm-ai-co-mathematician", "source_row": {"Best score (across scorers)": "0.756", "Country": "United States of America", "Model version": "gdm-ai-co-mathematician", "Organization": "Google DeepMind", "Release date": "2026-05-08", "Started at": "2026-06-12T14:01:58.023Z", "id": "gdm-ai-co-mathematician-manual-run-v2", "mean_score": "0.756", "stderr": "0.067"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-10-15", "model": "muse-wonder", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5416666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-wonder-1.0_max", "source_row": {"Best score (across scorers)": "0.5416666666666666", "Country": "United States of America", "Model version": "muse-wonder-1.0_max", "Organization": "Meta AI", "Release date": "2026-10-12", "Started at": "2026-10-15T22:49:11.000Z", "id": "iuNhWVj65sV54m3cKVwv9x", "mean_score": "0.5416666666666666", "stderr": "0.06432138878020231"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-10-14", "model": "muse-wonder", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.51, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-wonder-1.0_max", "source_row": {"Best score (across scorers)": "0.51", "Country": "United States of America", "Model version": "muse-wonder-1.0_max", "Organization": "Meta AI", "Release date": "2026-10-12", "Started at": "2026-10-14T16:17:18.000Z", "id": "gmscEuzbtZeUeFUFZMZiAV", "mean_score": "0.51", "stderr": "0.05024183937956913"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-10-12", "model": "muse-wonder", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7073170731707317, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-wonder-1.0_max", "source_row": {"Best score (across scorers)": "0.7073170731707317", "Country": "United States of America", "Model version": "muse-wonder-1.0_max", "Organization": "Meta AI", "Release date": "2026-10-12", "Started at": "2026-10-12T19:00:20.000Z", "id": "76KF9F9owpePuyHpP68mdZ", "mean_score": "0.7073170731707317", "stderr": "0.07194088392074452"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-10-12", "model": "muse-wonder", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8210526315789474, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-wonder-1.0_max", "source_row": {"Best score (across scorers)": "0.8210526315789474", "Country": "United States of America", "Model version": "muse-wonder-1.0_max", "Organization": "Meta AI", "Release date": "2026-10-12", "Started at": "2026-10-12T19:00:20.000Z", "id": "hwVgELmPofPY3Vqm7BRLxs", "mean_score": "0.8210526315789474", "stderr": "0.02274515950332741"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-10-12", "model": "muse-wonder", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9944444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-wonder-1.0_max", "source_row": {"Best score (across scorers)": "0.9944444444444445", "Country": "United States of America", "Model version": "muse-wonder-1.0_max", "Organization": "Meta AI", "Release date": "2026-10-12", "Started at": "2026-10-12T19:00:20.000Z", "id": "AFigcTGzpgZgMi6j2fpboa", "mean_score": "0.9944444444444445", "stderr": "0.003883473863441843"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "gpt-5-4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.59, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-pro-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.59", "Country": "United States of America", "Model version": "gpt-5.4-pro-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-10-14T09:56:24.000Z", "id": "5yc9mNtsSiFQ5PducfGZGT", "mean_score": "0.59", "stderr": "0.049431107042371025"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-13", "model": "gpt-5-4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5853658536585366, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-pro-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.5853658536585366", "Country": "United States of America", "Model version": "gpt-5.4-pro-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-06-13T02:38:02.000Z", "id": "W6b3C8AeNC7CAFs5DQma8N", "mean_score": "0.5853658536585366", "stderr": "0.07789619230571372"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-13", "model": "gpt-5-4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8245614035087719, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-pro-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.8245614035087719", "Country": "United States of America", "Model version": "gpt-5.4-pro-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-06-13T02:38:02.000Z", "id": "ZFVa8dw7oiir6PhcF8XZMo", "mean_score": "0.8245614035087719", "stderr": "0.022569134425265196"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-03-20", "model": "gpt-5-4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.946, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-pro-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.946", "Country": "United States of America", "Model version": "gpt-5.4-pro-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-03-20T00:19:39.820Z", "id": "gpt-5-4-pro-gpqa-cloud-run", "mean_score": "0.946", "stderr": "0.016"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gpt-5-4-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.463, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.4-pro-2026-03-05_xhigh", "source_row": {"Best score (across scorers)": "0.463", "Country": "United States of America", "Model version": "gpt-5.4-pro-2026-03-05_xhigh", "Organization": "OpenAI", "Release date": "2026-03-05", "Started at": "2026-08-27T19:28:24.000Z", "id": "RfSJbuuuMkeVHQj7xrth4u", "mean_score": "0.463", "stderr": "0.015775927227262426"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "gpt-5-2-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-pro-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.6", "Country": "United States of America", "Model version": "gpt-5.2-pro-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-10-14T09:56:34.000Z", "id": "LgRDo2HcNECAGiD8UnHYjx", "mean_score": "0.6", "stderr": "0.0492365963917331"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-13", "model": "gpt-5-2-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-pro-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.46", "Country": "United States of America", "Model version": "gpt-5.2-pro-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-06-13T02:37:55.000Z", "id": "oALBfwNc7dcNSDJSSBJXYe", "mean_score": "0.46", "stderr": "0.07789619230571372"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-13", "model": "gpt-5-2-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.74, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.2-pro-2025-12-11_xhigh", "source_row": {"Best score (across scorers)": "0.74", "Country": "United States of America", "Model version": "gpt-5.2-pro-2025-12-11_xhigh", "Organization": "OpenAI", "Release date": "2025-12-11", "Started at": "2026-06-13T02:37:55.000Z", "id": "dMJenf5xYWu7Pwn2U6w4dW", "mean_score": "0.74", "stderr": "0.026349537704451795"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "mimo-v2-6-pro", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8989897499999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.6-pro-20260921", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "rows": [{"accuracy": "0.8895694008908687", "provider_name": "DeepInfra", "total_questions": "898"}, {"accuracy": "0.8779042927756654", "provider_name": "GMICloud", "total_questions": "789"}, {"accuracy": "0.9023565", "provider_name": "Novita", "total_questions": "792"}, {"accuracy": "0.8989897499999999", "provider_name": "Xiaomi", "total_questions": "792"}], "rule": "first-party provider (1 of 4 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "mimo-v2-6-pro", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7983335000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.6-pro-20260921", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-pro-20260921", "rows": [{"accuracy": "0.7730925662650602", "provider_name": "DeepInfra", "total_questions": "249"}, {"accuracy": "0.7808335", "provider_name": "GMICloud", "total_questions": "200"}, {"accuracy": "0.7799996666666668", "provider_name": "Novita", "total_questions": "150"}, {"accuracy": "0.7983335000000001", "provider_name": "Xiaomi", "total_questions": "200"}], "rule": "first-party provider (1 of 4 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "qwen-3-8-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max_xhigh", "source_row": {"Best score (across scorers)": "0.46", "Country": "China", "Model version": "qwen3.8-max_xhigh", "Organization": "Alibaba", "Release date": "2026-08-02", "Started at": "2026-10-13T13:13:49.000Z", "id": "bq5y6Tp83KLtxC58UDfpyN", "mean_score": "0.46", "stderr": "0.05009082659620331"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-05", "model": "qwen-3-8-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.38, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max_xhigh", "source_row": {"Best score (across scorers)": "0.38", "Country": "China", "Model version": "qwen3.8-max_xhigh", "Organization": "Alibaba", "Release date": "2026-08-02", "Started at": "2026-08-05T01:42:38.000Z", "id": "c2YLifDU4zYCX5Jbi9HLL7", "mean_score": "0.38", "stderr": "0.048783173121456344"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-04", "model": "qwen-3-8-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4634146341463415, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max_xhigh", "source_row": {"Best score (across scorers)": "0.4634146341463415", "Country": "China", "Model version": "qwen3.8-max_xhigh", "Organization": "Alibaba", "Release date": "2026-08-02", "Started at": "2026-08-04T19:32:51.000Z", "id": "PBRdMZyz4L4CqPpXyLD2Lk", "mean_score": "0.4634146341463415", "stderr": "0.07884502378168105"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-04", "model": "qwen-3-8-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7473684210526316, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max_xhigh", "source_row": {"Best score (across scorers)": "0.7473684210526316", "Country": "China", "Model version": "qwen3.8-max_xhigh", "Organization": "Alibaba", "Release date": "2026-08-02", "Started at": "2026-08-04T19:32:51.000Z", "id": "ABT89ieSWGnGj4RYMduHiG", "mean_score": "0.7473684210526316", "stderr": "0.0257841025556124"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-04", "model": "qwen-3-8-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9944444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max_xhigh", "source_row": {"Best score (across scorers)": "0.9944444444444445", "Country": "China", "Model version": "qwen3.8-max_xhigh", "Organization": "Alibaba", "Release date": "2026-08-02", "Started at": "2026-08-04T16:28:46.000Z", "id": "aGRKHaq9uSawfYVpC565zU", "mean_score": "0.9944444444444445", "stderr": "0.0038834738634418433"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-04", "model": "qwen-3-8-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9267676767676768, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max_xhigh", "source_row": {"Best score (across scorers)": "0.9267676767676768", "Country": "China", "Model version": "qwen3.8-max_xhigh", "Organization": "Alibaba", "Release date": "2026-08-02", "Started at": "2026-08-04T16:28:46.000Z", "id": "TEVNDC9QXBh7PQvX9kHZip", "mean_score": "0.9267676767676768", "stderr": "0.01686498021450146"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen-3-8-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.458, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max_xhigh", "source_row": {"Best score (across scorers)": "0.458", "Country": "China", "Model version": "qwen3.8-max_xhigh", "Organization": "Alibaba", "Release date": "2026-08-02", "Started at": "2026-08-27T19:30:55.000Z", "id": "bfftd5EzUK9yAekafEofvK", "mean_score": "0.458", "stderr": "0.015763390640483554"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-08", "model": "deepseek-v4-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8989900000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-flash-20260423", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "rows": [{"accuracy": "0.8706306660808436", "provider_name": "Alibaba", "total_questions": "1707"}, {"accuracy": "0.8609005", "provider_name": "AtlasCloud", "total_questions": "1584"}, {"accuracy": "0.8532947142857142", "provider_name": "Azure", "total_questions": "1386"}, {"accuracy": "0.8621932857142858", "provider_name": "Baidu", "total_questions": "1386"}, {"accuracy": "0.8709067008816122", "provider_name": "Cloudflare", "total_questions": "1588"}, {"accuracy": "0.8601822310030396", "provider_name": "CoreWeave", "total_questions": "987"}, {"accuracy": "0.865880181448332", "provider_name": "DeepInfra", "total_questions": "1229"}, {"accuracy": "0.8989900000000001", "provider_name": "DeepSeek", "total_questions": "396"}, {"accuracy": "0.8089225000000001", "provider_name": "DigitalOcean", "total_questions": "1188"}, {"accuracy": "0.865101093922652", "provider_name": "Fireworks", "total_questions": "362"}, {"accuracy": "0.8905722857142856", "provider_name": "GMICloud", "total_questions": "1386"}, {"accuracy": "0.8619530000000001", "provider_name": "Ionstream", "total_questions": "198"}, {"accuracy": "0.8188131249999999", "provider_name": "Mancer 2", "total_questions": "1584"}, {"accuracy": "0.8653195", "provider_name": "NextBit", "total_questions": "396"}, {"accuracy": "0.8815751316455697", "provider_name": "Novita", "total_questions": "1185"}, {"accuracy": "0.8058380621468926", "provider_name": "OpenInference", "total_questions": "885"}, {"accuracy": "0.8698892857142857", "provider_name": "Parasail", "total_questions": "1386"}, {"accuracy": "0.8341984152249134", "provider_name": "Phala", "total_questions": "578"}, {"accuracy": "0.346801", "provider_name": "Reka", "total_questions": "198"}, {"accuracy": "0.8606902500000001", "provider_name": "Relace", "total_questions": "792"}, {"accuracy": "0.8692809647058825", "provider_name": "Sail Research", "total_questions": "765"}, {"accuracy": "0.8804712857142858", "provider_name": "SiliconFlow", "total_questions": "1386"}, {"accuracy": "0.8592169999999999", "provider_name": "StreamLake", "total_questions": "1584"}, {"accuracy": "0.839947", "provider_name": "Venice", "total_questions": "1386"}, {"accuracy": "0.877104", "provider_name": "Wafer", "total_questions": "396"}], "rule": "first-party provider (1 of 25 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-08", "model": "deepseek-v4-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7755553333333334, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v4-flash-20260423", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v4-flash-20260423", "rows": [{"accuracy": "0.7596315477386935", "provider_name": "Alibaba", "total_questions": "398"}, {"accuracy": "0.777778", "provider_name": "Ambient", "total_questions": "48"}, {"accuracy": "0.7562961111111112", "provider_name": "AtlasCloud", "total_questions": "450"}, {"accuracy": "0.7609522857142857", "provider_name": "Azure", "total_questions": "350"}, {"accuracy": "0.754166625", "provider_name": "Baidu", "total_questions": "400"}, {"accuracy": "0.7533333333333333", "provider_name": "Cloudflare", "total_questions": "450"}, {"accuracy": "0.738889", "provider_name": "CoreWeave", "total_questions": "300"}, {"accuracy": "0.7617447986577182", "provider_name": "DeepInfra", "total_questions": "298"}, {"accuracy": "0.7755553333333334", "provider_name": "DeepSeek", "total_questions": "150"}, {"accuracy": "0.766666625", "provider_name": "DigitalOcean", "total_questions": "400"}, {"accuracy": "0.7666665714285714", "provider_name": "GMICloud", "total_questions": "350"}, {"accuracy": "0.770833", "provider_name": "Io Net", "total_questions": "48"}, {"accuracy": "0.76", "provider_name": "Ionstream", "total_questions": "100"}, {"accuracy": "0.672916875", "provider_name": "Mancer 2", "total_questions": "400"}, {"accuracy": "0.75", "provider_name": "Morph", "total_questions": "100"}, {"accuracy": "0.7333330000000001", "provider_name": "NextBit", "total_questions": "100"}, {"accuracy": "0.757143", "provider_name": "Novita", "total_questions": "350"}, {"accuracy": "0.68666675", "provider_name": "OpenInference", "total_questions": "200"}, {"accuracy": "0.77583325", "provider_name": "Parasail", "total_questions": "400"}, {"accuracy": "0.7233335", "provider_name": "Phala", "total_questions": "100"}, {"accuracy": "0.773333", "provider_name": "Reka", "total_questions": "50"}, {"accuracy": "0.7583332500000001", "provider_name": "Relace", "total_questions": "200"}, {"accuracy": "0.7977776666666667", "provider_name": "Sail Research", "total_questions": "150"}, {"accuracy": "0.762500125", "provider_name": "SiliconFlow", "total_questions": "400"}, {"accuracy": "0.7522322901785714", "provider_name": "StreamLake", "total_questions": "448"}, {"accuracy": "0.7652466315789472", "provider_name": "Venice", "total_questions": "399"}, {"accuracy": "0.8", "provider_name": "Wafer", "total_questions": "100"}], "rule": "first-party provider (1 of 27 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "tencent-hy4-preview", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8973063333333335, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "tencent/hy4-preview-20260827", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "tencent/hy4-preview-20260827", "rows": [{"accuracy": "0.9052889151291513", "provider_name": "DeepInfra", "total_questions": "542"}, {"accuracy": "0.8934709999999999", "provider_name": "Makora", "total_questions": "194"}, {"accuracy": "0.89393925", "provider_name": "Novita", "total_questions": "792"}, {"accuracy": "0.8977269999999999", "provider_name": "SiliconFlow", "total_questions": "792"}, {"accuracy": "0.8973063333333335", "provider_name": "Tencent", "total_questions": "594"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "tencent-hy4-preview", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7777776666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "tencent/hy4-preview-20260827", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "tencent/hy4-preview-20260827", "rows": [{"accuracy": "0.777778", "provider_name": "DeepInfra", "total_questions": "150"}, {"accuracy": "0.746667", "provider_name": "InferenceNet", "total_questions": "50"}, {"accuracy": "0.773333", "provider_name": "Makora", "total_questions": "50"}, {"accuracy": "0.7688889999999999", "provider_name": "Novita", "total_questions": "150"}, {"accuracy": "0.7566667499999999", "provider_name": "SiliconFlow", "total_questions": "200"}, {"accuracy": "0.7777776666666667", "provider_name": "Tencent", "total_questions": "150"}], "rule": "first-party provider (1 of 6 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "muse-spark-1-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.3_max", "source_row": {"Best score (across scorers)": "0.37", "Country": "United States of America", "Model version": "muse-spark-1.3_max", "Organization": "Meta AI", "Release date": "2026-09-02", "Started at": "2026-10-13T13:00:28.000Z", "id": "XkHyFXCWYDv2nAB5vmmWrU", "mean_score": "0.37", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-18", "model": "muse-spark-1-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.25, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.3_max", "source_row": {"Best score (across scorers)": "0.25", "Country": "United States of America", "Model version": "muse-spark-1.3_max", "Organization": "Meta AI", "Release date": "2026-09-02", "Started at": "2026-09-18T19:12:25.000Z", "id": "6Rcc2q3USiUNB6U44bpvGH", "mean_score": "0.25", "stderr": "0.044640484731156636"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-18", "model": "muse-spark-1-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4634146341463415, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.3_max", "source_row": {"Best score (across scorers)": "0.4634146341463415", "Country": "United States of America", "Model version": "muse-spark-1.3_max", "Organization": "Meta AI", "Release date": "2026-09-02", "Started at": "2026-09-18T16:34:54.000Z", "id": "naZno5TPpgMJfv2b5mhvUW", "mean_score": "0.4634146341463415", "stderr": "0.07884502378168105"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-16", "model": "muse-spark-1-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.743859649122807, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.3_xhigh", "source_row": {"Best score (across scorers)": "0.743859649122807", "Country": "United States of America", "Model version": "muse-spark-1.3_xhigh", "Organization": "Meta AI", "Release date": "2026-09-02", "Started at": "2026-09-16T22:00:25.000Z", "id": "JeYksd3f2RkegmxCcZyBMQ", "mean_score": "0.743859649122807", "stderr": "0.025901524747773286"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-17", "model": "muse-spark-1-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9916666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.3_xhigh", "source_row": {"Best score (across scorers)": "0.9916666666666667", "Country": "United States of America", "Model version": "muse-spark-1.3_xhigh", "Organization": "Meta AI", "Release date": "2026-09-02", "Started at": "2026-09-17T13:30:27.000Z", "id": "Wmxkx5m3qaQuwzem6gzZmg", "mean_score": "0.9916666666666667", "stderr": "0.006154574548966636"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "qwen3-8-max-0902", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max-0902_xhigh", "source_row": {"Best score (across scorers)": "0.37", "Country": "China", "Model version": "qwen3.8-max-0902_xhigh", "Organization": "Alibaba", "Release date": "2026-09-01", "Started at": "2026-10-13T13:00:11.000Z", "id": "Ut4d4rvkUygdLTRomFb5rh", "mean_score": "0.37", "stderr": "0.048523658709390974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-10", "model": "qwen3-8-max-0902", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max-0902_xhigh", "source_row": {"Best score (across scorers)": "0.2", "Country": "China", "Model version": "qwen3.8-max-0902_xhigh", "Organization": "Alibaba", "Release date": "2026-09-01", "Started at": "2026-09-10T20:24:14.000Z", "id": "kpiLkPjDgoKRYp2SiKm9Zo", "mean_score": "0.2", "stderr": "0.05207556439232955"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-02", "model": "qwen3-8-max-0902", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34146341463414637, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max-0902_xhigh", "source_row": {"Best score (across scorers)": "0.34146341463414637", "Country": "China", "Model version": "qwen3.8-max-0902_xhigh", "Organization": "Alibaba", "Release date": "2026-09-01", "Started at": "2026-09-02T18:04:58.000Z", "id": "kZUdrqdGtCEJa3ip6L9YnH", "mean_score": "0.34146341463414637", "stderr": "0.07497768853141171"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-02", "model": "qwen3-8-max-0902", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.656140350877193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max-0902_xhigh", "source_row": {"Best score (across scorers)": "0.656140350877193", "Country": "China", "Model version": "qwen3.8-max-0902_xhigh", "Organization": "Alibaba", "Release date": "2026-09-01", "Started at": "2026-09-02T18:04:58.000Z", "id": "FpQf5xBwJPb4ZEo6SgdKWq", "mean_score": "0.656140350877193", "stderr": "0.02818576398907258"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-02", "model": "qwen3-8-max-0902", "retrieved_at": "2026-10-18T12:17:21Z", "score": 1.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max-0902_xhigh", "source_row": {"Best score (across scorers)": "1.0", "Country": "China", "Model version": "qwen3.8-max-0902_xhigh", "Organization": "Alibaba", "Release date": "2026-09-01", "Started at": "2026-09-02T18:04:58.000Z", "id": "2f8Vn5QBdJnc9i5y9jzXS7", "mean_score": "1.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-02", "model": "qwen3-8-max-0902", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.922979797979798, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max-0902_xhigh", "source_row": {"Best score (across scorers)": "0.922979797979798", "Country": "China", "Model version": "qwen3.8-max-0902_xhigh", "Organization": "Alibaba", "Release date": "2026-09-01", "Started at": "2026-09-02T18:04:58.000Z", "id": "VzeQXKBH9maR2nDcZJ8b7S", "mean_score": "0.922979797979798", "stderr": "0.017249976996605708"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-02", "model": "qwen3-8-max-0902", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.473, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.8-max-0902_xhigh", "source_row": {"Best score (across scorers)": "0.473", "Country": "China", "Model version": "qwen3.8-max-0902_xhigh", "Organization": "Alibaba", "Release date": "2026-09-01", "Started at": "2026-09-02T18:04:58.000Z", "id": "nBcLJJomVoeg2zh5F8Mnsu", "mean_score": "0.473", "stderr": "0.01579621855130273"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "muse-spark-1-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.1_xhigh", "source_row": {"Best score (across scorers)": "0.34", "Country": "United States of America", "Model version": "muse-spark-1.1_xhigh", "Organization": "Meta AI", "Release date": "2026-07-09", "Started at": "2026-10-13T13:13:33.000Z", "id": "4pbEvTxsbzNRCh9kU9Xhun", "mean_score": "0.34", "stderr": "0.04760952285695234"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "muse-spark-1-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5778894472361809, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.1", "source_row": {"Best score (across scorers)": "0.5778894472361809", "Country": "United States of America", "Model version": "muse-spark-1.1", "Organization": "Meta AI", "Release date": "2026-07-09", "Started at": "2026-08-31T19:21:58.000Z", "id": "bCdwpRXgZtvQLYC4wA9JXr", "mean_score": "0.5778894472361809", "stderr": "0.015665429072238852"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-10", "model": "qwen3-8-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8962022556962025, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.8-flash-20260826", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-flash-20260826", "rows": [{"accuracy": "0.8962022556962025", "provider_name": "Alibaba", "total_questions": "395"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "qwen3-8-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.8-flash-20260826", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-flash-20260826", "rows": [{"accuracy": "0.7", "provider_name": "Alibaba", "total_questions": "50"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "grok-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.35, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.5_high", "source_row": {"Best score (across scorers)": "0.35", "Country": "United States of America", "Model version": "grok-4.5_high", "Organization": "xAI", "Release date": "2026-07-08", "Started at": "2026-10-13T13:14:23.000Z", "id": "jTuYE8VJTos73Kw77UFevq", "mean_score": "0.35", "stderr": "0.047937248544110224"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-24", "model": "grok-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.225, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.5_high", "source_row": {"Best score (across scorers)": "0.225", "Country": "United States of America", "Model version": "grok-4.5_high", "Organization": "xAI", "Release date": "2026-07-08", "Started at": "2026-09-24T17:36:22.000Z", "id": "BFVFWpAaax3RDTXT7apuZE", "mean_score": "0.225", "stderr": "0.053711144147650755"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-07-09", "model": "grok-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.24390243902439024, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.5_high", "source_row": {"Best score (across scorers)": "0.24390243902439024", "Country": "United States of America", "Model version": "grok-4.5_high", "Organization": "xAI", "Release date": "2026-07-08", "Started at": "2026-07-09T01:53:42.000Z", "id": "7vThbrtbczScg2UcsAYXtF", "mean_score": "0.24390243902439024", "stderr": "0.06789956540036612"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-07-09", "model": "grok-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5719298245614035, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.5_high", "source_row": {"Best score (across scorers)": "0.5719298245614035", "Country": "United States of America", "Model version": "grok-4.5_high", "Organization": "xAI", "Release date": "2026-07-08", "Started at": "2026-07-09T01:53:42.000Z", "id": "djbbEqvBW9qux3pZwcZN2i", "mean_score": "0.5719298245614035", "stderr": "0.029360921879029937"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-07-08", "model": "grok-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9777777777777777, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.5_high", "source_row": {"Best score (across scorers)": "0.9777777777777777", "Country": "United States of America", "Model version": "grok-4.5_high", "Organization": "xAI", "Release date": "2026-07-08", "Started at": "2026-07-08T23:54:22.000Z", "id": "f8Pac6hqib4t7x2s4HCRoT", "mean_score": "0.9777777777777777", "stderr": "0.012743145865697103"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-07-08", "model": "grok-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9343434343434344, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.5_high", "source_row": {"Best score (across scorers)": "0.9343434343434344", "Country": "United States of America", "Model version": "grok-4.5_high", "Organization": "xAI", "Release date": "2026-07-08", "Started at": "2026-07-08T23:54:22.000Z", "id": "9kSgmUd9vh4CBXz3tJhctN", "mean_score": "0.9343434343434344", "stderr": "0.014340395485738813"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "grok-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.483, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.5_high", "source_row": {"Best score (across scorers)": "0.483", "Country": "United States of America", "Model version": "grok-4.5_high", "Organization": "xAI", "Release date": "2026-07-08", "Started at": "2026-08-27T19:29:30.000Z", "id": "7HWW3sjRstKzb9JpcnvLaV", "mean_score": "0.483", "stderr": "0.015810153729833274"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-14", "model": "gpt-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.48, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-pro-2025-10-06_high", "source_row": {"Best score (across scorers)": "0.48", "Country": "United States of America", "Model version": "gpt-5-pro-2025-10-06_high", "Organization": "OpenAI", "Release date": "2025-10-07", "Started at": "2026-10-14T09:56:39.000Z", "id": "RwHkGk6Qhd4NVTtRLDwLmc", "mean_score": "0.48", "stderr": "0.05021167315686782"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-12", "model": "gpt-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1951219512195122, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-pro-2025-10-06_high", "source_row": {"Best score (across scorers)": "0.1951219512195122", "Country": "United States of America", "Model version": "gpt-5-pro-2025-10-06_high", "Organization": "OpenAI", "Release date": "2025-10-07", "Started at": "2026-06-12T20:32:51.000Z", "id": "HfDahF4ZXTN6bjfDFwcHTU", "mean_score": "0.1951219512195122", "stderr": "0.06265967111543964"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-12", "model": "gpt-5-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5578947368421052, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5-pro-2025-10-06_high", "source_row": {"Best score (across scorers)": "0.5578947368421052", "Country": "United States of America", "Model version": "gpt-5-pro-2025-10-06_high", "Organization": "OpenAI", "Release date": "2025-10-07", "Started at": "2026-06-12T20:32:51.000Z", "id": "TFDQUeRKbWiEodRQRrKdpy", "mean_score": "0.5578947368421052", "stderr": "0.029469977528223976"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.41, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.41", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-10-13T12:39:21.000Z", "id": "L5Ck7pMy8bVmVMMLwxJN7f", "mean_score": "0.41", "stderr": "0.04943110704237103"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-09-24", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.20833333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.20833333333333334", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-09-24T16:31:21.000Z", "id": "jWTJKkY9kqYnkvZXCMKnwP", "mean_score": "0.20833333333333334", "stderr": "0.052199728697406675"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-09-22", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.29, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.29", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-09-22T21:49:22.000Z", "id": "Hk8bjr4u6KGqVE8SrPFKLq", "mean_score": "0.29", "stderr": "0.045604802157206865"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-09-22", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17073170731707318, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.17073170731707318", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-09-22T21:49:22.000Z", "id": "Ymgj5VeEceBbzsLq6xEZRi", "mean_score": "0.17073170731707318", "stderr": "0.05949419959829495"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-09-22", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5298245614035088, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.5298245614035088", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-09-22T21:49:22.000Z", "id": "F4unBbW5M4VJogzCq4vGAb", "mean_score": "0.5298245614035088", "stderr": "0.029616712041900365"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-09-22", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9805555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.9805555555555555", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-09-22T21:49:22.000Z", "id": "Jhgwm7QpspGcvm7DUVqUDg", "mean_score": "0.9805555555555555", "stderr": "0.014299486306926387"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-09-22", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.92739898989899, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.92739898989899", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-09-22T21:49:22.000Z", "id": "hxAhYJfXdPNanxatXpG5hC", "mean_score": "0.92739898989899", "stderr": "0.015505824295534454"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-09-22", "model": "grok-4-7", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.56, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.7_xhigh", "source_row": {"Best score (across scorers)": "0.56", "Country": "United States of America", "Model version": "grok-4.7_xhigh", "Organization": "xAI", "Release date": "2026-09-21", "Started at": "2026-09-22T21:49:22.000Z", "id": "6izDCw5ygCLZJEKPS8jV5T", "mean_score": "0.56", "stderr": "0.015704987954361718"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-11-19", "model": "gemini-3-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9138888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-pro-preview", "source_row": {"Best score (across scorers)": "0.9138888888888889", "Country": "United States of America", "Model version": "gemini-3-pro-preview", "Organization": "Google DeepMind", "Release date": "2025-11-18", "Started at": "2025-11-19T15:16:13.231Z", "id": "4QM4h7ndQiDRL2anjKeddK", "mean_score": "0.9138888888888889", "stderr": "0.03658872186755414"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-13", "model": "gemini-3-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7293388429752066, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-pro-preview", "source_row": {"Best score (across scorers)": "0.7293388429752066", "Country": "United States of America", "Model version": "gemini-3-pro-preview", "Organization": "Google DeepMind", "Release date": "2025-11-18", "Started at": "2026-02-13T10:43:27.233Z", "id": "5XdYZBG5Wy8DUMRqcYv4fn", "mean_score": "0.7293388429752066", "stderr": "0.02021641171153417"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-11-19", "model": "gemini-3-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9261363636363636, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3-pro-preview", "source_row": {"Best score (across scorers)": "0.9261363636363636", "Country": "United States of America", "Model version": "gemini-3-pro-preview", "Organization": "Google DeepMind", "Release date": "2025-11-18", "Started at": "2025-11-19T15:16:10.279Z", "id": "Cg54udQGcTCnfDGX2qePLZ", "mean_score": "0.9261363636363636", "stderr": "0.016526963574741"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gemini-3-pro", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.7251000000000001, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Gemini-3-Pro-Preview (Prompt)", "source_row": {"Model": "Gemini-3-Pro-Preview (Prompt)", "Organization": "Google", "Overall Acc": "72.51%", "Rank": "3"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-25", "model": "gpt-5-3-codex", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7479338842975206, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.3-codex_high", "source_row": {"Best score (across scorers)": "0.7479338842975206", "Country": "United States of America", "Model version": "gpt-5.3-codex_high", "Organization": "OpenAI", "Release date": "2026-02-05", "Started at": "2026-02-25T10:37:06.792Z", "id": "6w5buMbgeEqtRoE55cLZVv", "mean_score": "0.7479338842975206", "stderr": "0.019756739501473306"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "gpt-5-3-codex", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.753333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-5.3-codex-20260224", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-5.3-codex-20260224", "rows": [{"accuracy": "0.753333", "provider_name": "OpenAI", "total_questions": "50"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-15", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.33, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.33", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-15T22:52:59.000Z", "id": "eNdDCt2u6A8mPisK56SApU", "mean_score": "0.33", "stderr": "0.04725815626252608"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "furniture_assembly", "eval_date": "2026-10-14", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.20833333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.20833333333333334", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-14T00:39:04.000Z", "id": "cETggiScYbsKGvPW5BAVWJ", "mean_score": "0.20833333333333334", "stderr": "0.052199728697406675"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#furniture_assembly.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-10-13", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.17", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-13T22:01:32.000Z", "id": "LT9xLJnaU4UHsBQrvu3rbU", "mean_score": "0.17", "stderr": "0.03775251680686369"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-10-12", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2682926829268293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.2682926829268293", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-12T21:38:35.000Z", "id": "TzoNRjKto6wEuZ5PmQPAw8", "mean_score": "0.2682926829268293", "stderr": "0.07005564203095158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-10-12", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5859649122807018, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.5859649122807018", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-12T21:38:35.000Z", "id": "CAstp8hQG6Aehvtw85bMDX", "mean_score": "0.5859649122807018", "stderr": "0.02922773823944577"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-10-12", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9833333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.9833333333333333", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-12T21:38:35.000Z", "id": "Pby5WuP8Muu6Jzs2v5gB7F", "mean_score": "0.9833333333333333", "stderr": "0.006405884289881728"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-10-12", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8806818181818182, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.8806818181818182", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-12T21:38:22.000Z", "id": "Ekx5phukyiJhAcQodMfS2p", "mean_score": "0.8806818181818182", "stderr": "0.019328132475292338"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-10-09", "model": "mistral-large-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-4-0_high", "source_row": {"Best score (across scorers)": "0.2", "Country": "France", "Model version": "mistral-large-4-0_high", "Organization": "Mistral AI", "Release date": "2026-10-06", "Started at": "2026-10-09T14:12:09.000Z", "id": "hvNNKdnzLvr9X8zVxWKcfk", "mean_score": "0.2", "stderr": "0.012655439943366657"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "mimo-v2-5-pro", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8550657272727274, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.5-pro-20260422", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "rows": [{"accuracy": "0.6637718184210527", "provider_name": "AtlasCloud", "total_questions": "1900"}, {"accuracy": "0.8214198161925602", "provider_name": "DeepInfra", "total_questions": "1371"}, {"accuracy": "0.8191437142857143", "provider_name": "DigitalOcean", "total_questions": "1386"}, {"accuracy": "0.7975339581632652", "provider_name": "GMICloud", "total_questions": "1960"}, {"accuracy": "0.8507279006896552", "provider_name": "Novita", "total_questions": "2175"}, {"accuracy": "0.861993011958763", "provider_name": "StreamLake", "total_questions": "2425"}, {"accuracy": "0.8550657272727274", "provider_name": "Xiaomi", "total_questions": "2178"}], "rule": "first-party provider (1 of 7 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "mimo-v2-5-pro", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.743889, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.5-pro-20260422", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-pro-20260422", "rows": [{"accuracy": "0.5935779504587155", "provider_name": "AtlasCloud", "total_questions": "545"}, {"accuracy": "0.7284644516853932", "provider_name": "DeepInfra", "total_questions": "445"}, {"accuracy": "0.6706667", "provider_name": "DigitalOcean", "total_questions": "500"}, {"accuracy": "0.7405555", "provider_name": "Novita", "total_questions": "600"}, {"accuracy": "0.7382021797752809", "provider_name": "StreamLake", "total_questions": "445"}, {"accuracy": "0.743889", "provider_name": "Xiaomi", "total_questions": "600"}], "rule": "first-party provider (1 of 6 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "muse-spark-1-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.27, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.2_xhigh", "source_row": {"Best score (across scorers)": "0.27", "Country": "United States of America", "Model version": "muse-spark-1.2_xhigh", "Organization": "Meta AI", "Release date": "2026-08-05", "Started at": "2026-10-13T13:05:03.000Z", "id": "FueK5bFAzGbkUao9JAibms", "mean_score": "0.27", "stderr": "0.044619604333847374"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "muse-spark-1-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6030150753768844, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark-1.2_xhigh", "source_row": {"Best score (across scorers)": "0.6030150753768844", "Country": "United States of America", "Model version": "muse-spark-1.2_xhigh", "Organization": "Meta AI", "Release date": "2026-08-05", "Started at": "2026-08-27T19:36:49.000Z", "id": "mLLmVVvo6t23MM5yxq7ryW", "mean_score": "0.6030150753768844", "stderr": "0.015518791563606454"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "minimax-m2-7", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8570973081339713, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m2.7-20260318", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.7-20260318", "rows": [{"accuracy": "0.8548968432122371", "provider_name": "AtlasCloud", "total_questions": "1569"}, {"accuracy": "0.7631440720576461", "provider_name": "DeepInfra", "total_questions": "1249"}, {"accuracy": "0.8463673576923078", "provider_name": "GMICloud", "total_questions": "1560"}, {"accuracy": "0.7796947507042253", "provider_name": "Groq", "total_questions": "1420"}, {"accuracy": "0.7885339041353384", "provider_name": "Mara", "total_questions": "532"}, {"accuracy": "0.8570973081339713", "provider_name": "Minimax", "total_questions": "3135"}, {"accuracy": "0.8504434125475285", "provider_name": "Novita", "total_questions": "1578"}, {"accuracy": "0.858586", "provider_name": "PrimeIntellect", "total_questions": "198"}, {"accuracy": "0.7805031924528302", "provider_name": "SambaNova", "total_questions": "265"}], "rule": "first-party provider (1 of 9 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "minimax-m2-7", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7203921764705883, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m2.7-20260318", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.7-20260318", "rows": [{"accuracy": "0.709166875", "provider_name": "AtlasCloud", "total_questions": "400"}, {"accuracy": "0.7012261270903011", "provider_name": "DeepInfra", "total_questions": "299"}, {"accuracy": "0.7229628888888889", "provider_name": "GMICloud", "total_questions": "450"}, {"accuracy": "0.727316106017192", "provider_name": "Groq", "total_questions": "349"}, {"accuracy": "0.7139639594594595", "provider_name": "Mara", "total_questions": "148"}, {"accuracy": "0.7203921764705883", "provider_name": "Minimax", "total_questions": "850"}, {"accuracy": "0.6933333333333334", "provider_name": "Novita", "total_questions": "450"}, {"accuracy": "0.7", "provider_name": "PrimeIntellect", "total_questions": "50"}, {"accuracy": "0.709172389261745", "provider_name": "SambaNova", "total_questions": "149"}], "rule": "first-party provider (1 of 9 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "minimax-m2-1", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.868687, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m2.1", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.1", "rows": [{"accuracy": "0.868687", "provider_name": "Novita", "run_count": "1", "total_questions": "198"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "minimax-m2-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6783334999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m2.1", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.1", "rows": [{"accuracy": "0.6783334999999999", "provider_name": "Minimax", "total_questions": "200"}, {"accuracy": "0.633333", "provider_name": "Novita", "total_questions": "100"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "qwen-3-8-27b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.83367, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.8-27b-20260814", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.8-27b-20260814", "rows": [{"accuracy": "0.8237281106367841", "provider_name": "AkashML", "total_questions": "2811"}, {"accuracy": "0.83367", "provider_name": "Alibaba", "total_questions": "1980"}, {"accuracy": "0.8316496666666667", "provider_name": "Cerebras", "total_questions": "594"}, {"accuracy": "0.8232322999999999", "provider_name": "Chutes", "total_questions": "1980"}, {"accuracy": "0.8378307503892062", "provider_name": "Cloudflare", "total_questions": "1927"}, {"accuracy": "0.8197122727272727", "provider_name": "CoreWeave", "total_questions": "2178"}, {"accuracy": "0.81607725", "provider_name": "Darkbloom", "total_questions": "1584"}, {"accuracy": "0.8356163613013698", "provider_name": "DeepInfra", "total_questions": "1168"}, {"accuracy": "0.8345362173215719", "provider_name": "DekaLLM", "total_questions": "1247"}, {"accuracy": "0.7950336666666666", "provider_name": "Io Net", "total_questions": "1188"}, {"accuracy": "0.8274411249999999", "provider_name": "Ionstream", "total_questions": "1584"}, {"accuracy": "0.8186202474674383", "provider_name": "Mancer 2", "total_questions": "1382"}, {"accuracy": "0.8033669999999999", "provider_name": "ModelRun", "total_questions": "990"}, {"accuracy": "0.8075196666666667", "provider_name": "Near AI", "total_questions": "594"}, {"accuracy": "0.8264122222222224", "provider_name": "Novita", "total_questions": "1782"}, {"accuracy": "0.8302596014344261", "provider_name": "Parasail", "total_questions": "1952"}, {"accuracy": "0.7467264444444446", "provider_name": "Phala", "total_questions": "1782"}, {"accuracy": "0.7676531051401869", "provider_name": "Reka", "total_questions": "2568"}, {"accuracy": "0.806818", "provider_name": "Tenstorrent", "total_questions": "792"}, {"accuracy": "0.7919196568825911", "provider_name": "Venice", "total_questions": "1976"}, {"accuracy": "0.8305271666666666", "provider_name": "Wafer", "total_questions": "1188"}], "rule": "first-party provider (1 of 21 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "qwen-3-8-27b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7592592222222222, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.8-27b-20260814", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.8-27b-20260814", "rows": [{"accuracy": "0.7737493322091062", "provider_name": "AkashML", "total_questions": "593"}, {"accuracy": "0.7592592222222222", "provider_name": "Alibaba", "total_questions": "450"}, {"accuracy": "0.8083334999999999", "provider_name": "Cerebras", "total_questions": "100"}, {"accuracy": "0.7718517777777778", "provider_name": "Chutes", "total_questions": "450"}, {"accuracy": "0.7696296666666667", "provider_name": "Cloudflare", "total_questions": "450"}, {"accuracy": "0.7566667", "provider_name": "CoreWeave", "total_questions": "500"}, {"accuracy": "0.7534247123287672", "provider_name": "Darkbloom", "total_questions": "146"}, {"accuracy": "0.781111", "provider_name": "DeepInfra", "total_questions": "300"}, {"accuracy": "0.7752381428571429", "provider_name": "DekaLLM", "total_questions": "350"}, {"accuracy": "0.7733335", "provider_name": "Io Net", "total_questions": "100"}, {"accuracy": "0.758189028901734", "provider_name": "Ionstream", "total_questions": "346"}, {"accuracy": "0.7590475714285715", "provider_name": "Mancer 2", "total_questions": "350"}, {"accuracy": "0.7853332000000001", "provider_name": "ModelRun", "total_questions": "250"}, {"accuracy": "0.7844443333333334", "provider_name": "Near AI", "total_questions": "150"}, {"accuracy": "0.7625", "provider_name": "Novita", "total_questions": "400"}, {"accuracy": "0.7740742222222222", "provider_name": "Parasail", "total_questions": "450"}, {"accuracy": "0.734166625", "provider_name": "Phala", "total_questions": "400"}, {"accuracy": "0.7607481658291457", "provider_name": "Reka", "total_questions": "597"}, {"accuracy": "0.66166675", "provider_name": "Tenstorrent", "total_questions": "200"}, {"accuracy": "0.71999975", "provider_name": "Venice", "total_questions": "400"}, {"accuracy": "0.7588888333333333", "provider_name": "Wafer", "total_questions": "300"}], "rule": "first-party provider (1 of 21 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "aider_polyglot", "eval_date": "2025-06-28", "model": "o3-pro", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.8490000000000001, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "o3-pro (high)", "source_row": {"command": "aider --model o3-pro", "date": "2025-06-28", "dirname": "2025-06-28-00-38-18--o3-pro-high", "model": "o3-pro (high)", "pass_rate_2": 84.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-29", "model": "minimax-m2-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8539973475609757, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m2.5-20260211", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "minimax/minimax-m2.5-20260211", "rows": [{"accuracy": "0.7950015744680852", "provider_name": "AtlasCloud", "total_questions": "987"}, {"accuracy": "0.8174118475609755", "provider_name": "DigitalOcean", "total_questions": "984"}, {"accuracy": "0.8160686215384615", "provider_name": "Friendli", "total_questions": "975"}, {"accuracy": "0.8409475076142132", "provider_name": "Inceptron", "total_questions": "591"}, {"accuracy": "0.8539973475609757", "provider_name": "Minimax", "total_questions": "1968"}, {"accuracy": "0.599327", "provider_name": "Nebius", "total_questions": "396"}, {"accuracy": "0.831313", "provider_name": "Novita", "total_questions": "990"}, {"accuracy": "0.8147209999999999", "provider_name": "Parasail", "total_questions": "197"}, {"accuracy": "0.8419978532110092", "provider_name": "SiliconFlow", "total_questions": "981"}, {"accuracy": "0.8239056", "provider_name": "StreamLake", "total_questions": "990"}, {"accuracy": "0.6995251282660333", "provider_name": "Venice", "total_questions": "842"}], "rule": "first-party provider (1 of 11 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-29", "model": "minimax-m2-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6905556666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "minimax/minimax-m2.5-20260211", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "minimax/minimax-m2.5-20260211", "rows": [{"accuracy": "0.6299998333333334", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.63", "provider_name": "CoreWeave", "total_questions": "100"}, {"accuracy": "0.6622221666666667", "provider_name": "DigitalOcean", "total_questions": "300"}, {"accuracy": "0.6644445", "provider_name": "Friendli", "total_questions": "300"}, {"accuracy": "0.64083325", "provider_name": "Inceptron", "total_questions": "200"}, {"accuracy": "0.6905556666666667", "provider_name": "Minimax", "total_questions": "600"}, {"accuracy": "0.5333335", "provider_name": "Nebius", "total_questions": "100"}, {"accuracy": "0.6455555000000001", "provider_name": "Novita", "total_questions": "300"}, {"accuracy": "0.619048", "provider_name": "Parasail", "total_questions": "49"}, {"accuracy": "0.656", "provider_name": "SiliconFlow", "total_questions": "250"}, {"accuracy": "0.6455555000000001", "provider_name": "StreamLake", "total_questions": "300"}, {"accuracy": "0.6949152745762711", "provider_name": "Venice", "total_questions": "295"}], "rule": "first-party provider (1 of 12 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-04-08", "model": "muse-spark", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark", "source_row": {"Best score (across scorers)": "0.889", "Country": "United States of America", "Model version": "muse-spark", "Organization": "Meta AI", "Release date": "2026-04-08", "Started at": "2026-04-08T16:30:04.184Z", "id": "muse-spark-pre-release-mock-aime", "mean_score": "0.889", "stderr": "0.047"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-04-08", "model": "muse-spark", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.898, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "muse-spark", "source_row": {"Best score (across scorers)": "0.898", "Country": "United States of America", "Model version": "muse-spark", "Organization": "Meta AI", "Release date": "2026-04-08", "Started at": "2026-04-08T16:29:51.099Z", "id": "muse-spark-pre-release-gpqa", "mean_score": "0.898", "stderr": "0.022"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen-3-6-max-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.19, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-max-preview_none", "source_row": {"Best score (across scorers)": "0.19", "Country": "China", "Model version": "qwen3.6-max-preview_none", "Organization": "Alibaba", "Release date": "2026-04-20", "Started at": "2026-08-27T19:13:41.000Z", "id": "mbySBWyYrKdtd7joxHyBdq", "mean_score": "0.19", "stderr": "0.039427724440366255"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen-3-6-max-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-max-preview", "source_row": {"Best score (across scorers)": "0.9111111111111111", "Country": "China", "Model version": "qwen3.6-max-preview", "Organization": "Alibaba", "Release date": "2026-04-20", "Started at": "2026-08-07T00:42:24.000Z", "id": "EfJpx5HztMmL5EQSwsyqMe", "mean_score": "0.9111111111111111", "stderr": "0.04290254662948546"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-05-28", "model": "qwen-3-6-max-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7665289256198347, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-max-preview", "source_row": {"Best score (across scorers)": "0.7665289256198347", "Country": "China", "Model version": "qwen3.6-max-preview", "Organization": "Alibaba", "Release date": "2026-04-20", "Started at": "2026-05-28T10:24:53.000Z", "id": "FkDMVLdEnJDR5PyUCT6BNQ", "mean_score": "0.7665289256198347", "stderr": "0.01924895909142592"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen-3-6-max-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8737373737373737, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-max-preview", "source_row": {"Best score (across scorers)": "0.8737373737373737", "Country": "China", "Model version": "qwen3.6-max-preview", "Organization": "Alibaba", "Release date": "2026-04-20", "Started at": "2026-08-07T00:46:39.000Z", "id": "Jv2hJvjpvA8xJW3Bzv7bfw", "mean_score": "0.8737373737373737", "stderr": "0.023664359402880177"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen-3-6-max-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5200400801603207, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-max-preview", "source_row": {"Best score (across scorers)": "0.5200400801603207", "Country": "China", "Model version": "qwen3.6-max-preview", "Organization": "Alibaba", "Release date": "2026-04-20", "Started at": "2026-08-27T19:33:58.000Z", "id": "iyPiR8GUuyNNA6Fssziq32", "mean_score": "0.5200400801603207", "stderr": "0.015822434816352416"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "mimo-v2-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8239975454545455, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.5-20260422", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "rows": [{"accuracy": "0.5948373333333333", "provider_name": "DeepInfra", "total_questions": "594"}, {"accuracy": "0.75076231097561", "provider_name": "GMICloud", "total_questions": "1968"}, {"accuracy": "0.8223140000000001", "provider_name": "Novita", "total_questions": "2178"}, {"accuracy": "0.6470586811145511", "provider_name": "Parasail", "total_questions": "646"}, {"accuracy": "0.8177376144306652", "provider_name": "StreamLake", "total_questions": "1774"}, {"accuracy": "0.7854672791234141", "provider_name": "Venice", "total_questions": "867"}, {"accuracy": "0.8239975454545455", "provider_name": "Xiaomi", "total_questions": "2178"}], "rule": "first-party provider (1 of 7 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "mimo-v2-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7372220833333334, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.5-20260422", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.5-20260422", "rows": [{"accuracy": "0.5271991666666667", "provider_name": "DeepInfra", "total_questions": "288"}, {"accuracy": "0.7354165", "provider_name": "GMICloud", "total_questions": "400"}, {"accuracy": "0.7416668333333333", "provider_name": "Novita", "total_questions": "600"}, {"accuracy": "0.72", "provider_name": "Parasail", "total_questions": "50"}, {"accuracy": "0.7366668", "provider_name": "Venice", "total_questions": "250"}, {"accuracy": "0.7372220833333334", "provider_name": "Xiaomi", "total_questions": "600"}], "rule": "first-party provider (1 of 6 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "grok-4-3-beta", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.18, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.3_high", "source_row": {"Best score (across scorers)": "0.18", "Country": "United States of America", "Model version": "grok-4.3_high", "Organization": "xAI", "Release date": "2026-04-17", "Started at": "2026-10-13T13:34:31.000Z", "id": "m4WQuAN88hKsqbT9zXbg7z", "mean_score": "0.18", "stderr": "0.03861229196653691"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-17", "model": "grok-4-3-beta", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14634146341463414, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.3_high", "source_row": {"Best score (across scorers)": "0.14634146341463414", "Country": "United States of America", "Model version": "grok-4.3_high", "Organization": "xAI", "Release date": "2026-04-17", "Started at": "2026-06-17T17:10:48.000Z", "id": "3iRQkf5v7Wu7YuZhxa76Gg", "mean_score": "0.14634146341463414", "stderr": "0.055885069450680974"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-17", "model": "grok-4-3-beta", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4280701754385965, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.3_high", "source_row": {"Best score (across scorers)": "0.4280701754385965", "Country": "United States of America", "Model version": "grok-4.3_high", "Organization": "xAI", "Release date": "2026-04-17", "Started at": "2026-06-17T17:10:48.000Z", "id": "bdsZjrG4ayzP8NAJg5q586", "mean_score": "0.4280701754385965", "stderr": "0.029360921879029937"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-06-17", "model": "grok-4-3-beta", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.3_high", "source_row": {"Best score (across scorers)": "0.9333333333333333", "Country": "United States of America", "Model version": "grok-4.3_high", "Organization": "xAI", "Release date": "2026-04-17", "Started at": "2026-06-17T17:51:54.000Z", "id": "fzoUFmy77NpEDtXyA6w6ap", "mean_score": "0.9333333333333333", "stderr": "0.03041173683586219"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-06-17", "model": "grok-4-3-beta", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8882575757575758, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.3_high", "source_row": {"Best score (across scorers)": "0.8882575757575758", "Country": "United States of America", "Model version": "grok-4.3_high", "Organization": "xAI", "Release date": "2026-04-17", "Started at": "2026-06-17T16:44:33.000Z", "id": "Vw7spGUnJyf3vuzSjTcj2F", "mean_score": "0.8882575757575758", "stderr": "0.01963840420048217"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "grok-4-3-beta", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.332, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4.3_high", "source_row": {"Best score (across scorers)": "0.332", "Country": "United States of America", "Model version": "grok-4.3_high", "Organization": "xAI", "Release date": "2026-04-17", "Started at": "2026-08-27T19:29:33.000Z", "id": "Y39ZRp553ULpUDBV2XrBbc", "mean_score": "0.332", "stderr": "0.014899597242811567"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-29", "model": "glm-4-6", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.7646065075987841, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-4.6", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.6", "rows": [{"accuracy": "0.7646065075987841", "provider_name": "AtlasCloud", "run_count": "5", "total_questions": "987"}, {"accuracy": "0.8203461428571428", "provider_name": "Novita", "run_count": "7", "total_questions": "1386"}, {"accuracy": "0.6643207216890595", "provider_name": "Venice", "run_count": "8", "total_questions": "1563"}], "rule": "median over providers (3 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "glm-4-6", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.7238, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "GLM-4.6 (FC thinking)", "source_row": {"Model": "GLM-4.6 (FC thinking)", "Organization": "Zhipu AI", "Overall Acc": "72.38%", "Rank": "4"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "glm-4-6", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7344445, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-4.6", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.6", "rows": [{"accuracy": "0.2066666", "provider_name": "AtlasCloud", "total_questions": "250"}, {"accuracy": "0.7066664", "provider_name": "DeepInfra", "total_questions": "250"}, {"accuracy": "0.73416675", "provider_name": "Novita", "total_questions": "400"}, {"accuracy": "0.6986607901785714", "provider_name": "Venice", "total_questions": "448"}, {"accuracy": "0.7344445", "provider_name": "Z.AI", "total_questions": "300"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "glm-4-5-air", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.68, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-4.5-air", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.5-air", "rows": [{"accuracy": "0.69", "provider_name": "Novita", "total_questions": "100"}, {"accuracy": "0.673333", "provider_name": "SiliconFlow", "total_questions": "50"}, {"accuracy": "0.68", "provider_name": "Z.AI", "total_questions": "50"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "deepseek-v3-2", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.802714625, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v3.2-20251201", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "rows": [{"accuracy": "0.823296396946565", "provider_name": "Alibaba", "run_count": "6", "total_questions": "1179"}, {"accuracy": "0.8236161787072243", "provider_name": "AtlasCloud", "run_count": "8", "total_questions": "1578"}, {"accuracy": "0.8278469506242907", "provider_name": "Baidu", "run_count": "9", "total_questions": "1762"}, {"accuracy": "0.8185119845094664", "provider_name": "DigitalOcean", "run_count": "9", "total_questions": "1743"}, {"accuracy": "0.802714625", "provider_name": "Friendli", "run_count": "8", "total_questions": "1584"}, {"accuracy": "0.7385620777988616", "provider_name": "GMICloud", "run_count": "8", "total_questions": "1581"}, {"accuracy": "0.707752053820598", "provider_name": "Google", "run_count": "8", "total_questions": "1505"}, {"accuracy": "0.7956228", "provider_name": "Novita", "run_count": "5", "total_questions": "990"}, {"accuracy": "0.7584345607287448", "provider_name": "Phala", "run_count": "5", "total_questions": "988"}, {"accuracy": "0.8168566996197718", "provider_name": "SiliconFlow", "run_count": "8", "total_questions": "1578"}, {"accuracy": "0.8124578000000001", "provider_name": "StreamLake", "run_count": "5", "total_questions": "990"}, {"accuracy": "0.7771783756345177", "provider_name": "Venice", "run_count": "8", "total_questions": "1576"}], "rule": "median over providers (12 of 12 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-04", "model": "deepseek-v3-2", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7193332, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v3.2-20251201", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-20251201", "rows": [{"accuracy": "0.7193332", "provider_name": "Alibaba", "total_questions": "250"}, {"accuracy": "0.664166625", "provider_name": "AtlasCloud", "total_questions": "400"}, {"accuracy": "0.7452106666666666", "provider_name": "DeepInfra", "total_questions": "348"}, {"accuracy": "0.7259258888888889", "provider_name": "Friendli", "total_questions": "450"}, {"accuracy": "0.6333331666666667", "provider_name": "GMICloud", "total_questions": "300"}, {"accuracy": "0.6415203403508771", "provider_name": "Google", "total_questions": "285"}, {"accuracy": "0.7128572857142856", "provider_name": "Phala", "total_questions": "350"}, {"accuracy": "0.7390475714285715", "provider_name": "SiliconFlow", "total_questions": "350"}, {"accuracy": "0.7400003333333333", "provider_name": "StreamLake", "total_questions": "300"}, {"accuracy": "0.7387772808022923", "provider_name": "Venice", "total_questions": "349"}], "rule": "median over providers (10 of 10 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "nemotron-3-ultra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "nemotron-3-ultra", "source_row": {"Best score (across scorers)": "0.2", "Country": "United States of America", "Model version": "nemotron-3-ultra", "Organization": "Nvidia", "Release date": "2026-06-04", "Started at": "2026-08-27T18:00:58.000Z", "Training compute (FLOP)": "6.6e+24", "id": "U8H8juE3Ko83jsR6TRntbh", "mean_score": "0.2", "stderr": "0.04020151261036849"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-10", "model": "nemotron-3-ultra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "nemotron-3-ultra", "source_row": {"Best score (across scorers)": "0.8666666666666667", "Country": "United States of America", "Model version": "nemotron-3-ultra", "Organization": "Nvidia", "Release date": "2026-06-04", "Started at": "2026-08-10T17:47:10.000Z", "Training compute (FLOP)": "6.6e+24", "id": "WYTWr3Xk4qxymhMsm7vRQ2", "mean_score": "0.8666666666666667", "stderr": "0.051247074319053836"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-10", "model": "nemotron-3-ultra", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8535353535353535, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "nemotron-3-ultra", "source_row": {"Best score (across scorers)": "0.8535353535353535", "Country": "United States of America", "Model version": "nemotron-3-ultra", "Organization": "Nvidia", "Release date": "2026-06-04", "Started at": "2026-08-10T17:46:52.000Z", "Training compute (FLOP)": "6.6e+24", "id": "bpZy8tAbh7UV6A6U7XZtBg", "mean_score": "0.8535353535353535", "stderr": "0.025190921114603887"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-04", "model": "nemotron-3-ultra", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.72, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "nvidia/nemotron-3-ultra-550b-a55b-20260604", "rows": [{"accuracy": "0.72", "provider_name": "BaseTen", "total_questions": "50"}, {"accuracy": "0.76", "provider_name": "Venice", "total_questions": "50"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "", "model": "grok-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.84, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4-0709", "source_row": {"Best score (across scorers)": "0.84", "Country": "United States of America", "Model version": "grok-4-0709", "Organization": "xAI", "Release date": "2025-07-09", "Training compute (FLOP)": "5.0000000000001e+26", "id": "cvTPRDCM38zSTn9Y3MUb9d", "mean_score": "0.84", "stderr": "0.05"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-07-11", "model": "grok-4", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.7959999999999999, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "grok-4 (high)", "source_row": {"command": "aider --model openrouter/x-ai/grok-4", "date": "2025-07-11", "dirname": "2025-07-11-19-37-40--xai-or-grok4-high", "model": "grok-4 (high)", "pass_rate_2": 79.6, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "", "model": "grok-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.87, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-4-0709", "source_row": {"Best score (across scorers)": "0.87", "Country": "United States of America", "Model version": "grok-4-0709", "Organization": "xAI", "Release date": "2025-07-09", "Training compute (FLOP)": "5.0000000000001e+26", "id": "A85Zfq2qguE4X9xXBweBHP", "mean_score": "0.87", "stderr": "0.02"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "grok-4", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.6297, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Grok-4-0709 (Prompt)", "source_row": {"Model": "Grok-4-0709 (Prompt)", "Organization": "xAI", "Overall Acc": "62.97%", "Rank": "9"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "qwen-3-6-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-plus", "source_row": {"Best score (across scorers)": "0.14", "Country": "China", "Model version": "qwen3.6-plus", "Organization": "Alibaba", "Release date": "2026-03-31", "Started at": "2026-10-13T13:38:57.000Z", "id": "SP9UFPzxVnsbzs2kExP2bh", "mean_score": "0.14", "stderr": "0.0348735088019777"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen-3-6-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-plus_none", "source_row": {"Best score (across scorers)": "0.12", "Country": "China", "Model version": "qwen3.6-plus_none", "Organization": "Alibaba", "Release date": "2026-03-31", "Started at": "2026-08-27T18:27:14.000Z", "id": "jGgkDRHbodcaTzQ3Mp6SxS", "mean_score": "0.12", "stderr": "0.03265986323710904"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-30", "model": "qwen-3-6-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3824561403508772, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-plus", "source_row": {"Best score (across scorers)": "0.3824561403508772", "Country": "China", "Model version": "qwen3.6-plus", "Organization": "Alibaba", "Release date": "2026-03-31", "Started at": "2026-08-30T22:34:20.000Z", "id": "jbyVWnEvvFVNNc8AEF6WcQ", "mean_score": "0.3824561403508772", "stderr": "0.028838027580159496"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen-3-6-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-plus", "source_row": {"Best score (across scorers)": "0.9333333333333333", "Country": "China", "Model version": "qwen3.6-plus", "Organization": "Alibaba", "Release date": "2026-03-31", "Started at": "2026-08-07T00:16:48.000Z", "id": "jYiGAFfUQKJuRStPzU3PX3", "mean_score": "0.9333333333333333", "stderr": "0.03760507165451774"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-05-14", "model": "qwen-3-6-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5785123966942148, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-plus", "source_row": {"Best score (across scorers)": "0.5785123966942148", "Country": "China", "Model version": "qwen3.6-plus", "Organization": "Alibaba", "Release date": "2026-03-31", "Started at": "2026-05-14T17:28:26.954Z", "id": "8HV4HP9EM5KuLSwzifuLye", "mean_score": "0.5785123966942148", "stderr": "0.02246855652298021"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen-3-6-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8838383838383839, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-plus", "source_row": {"Best score (across scorers)": "0.8838383838383839", "Country": "China", "Model version": "qwen3.6-plus", "Organization": "Alibaba", "Release date": "2026-03-31", "Started at": "2026-08-07T00:31:39.000Z", "id": "f6ZdMtRpdfskGbrBCzrKSn", "mean_score": "0.8838383838383839", "stderr": "0.022828881775249357"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen-3-6-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.44144144144144143, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-plus", "source_row": {"Best score (across scorers)": "0.44144144144144143", "Country": "China", "Model version": "qwen3.6-plus", "Organization": "Alibaba", "Release date": "2026-03-31", "Started at": "2026-08-27T19:34:06.000Z", "id": "4LWNorP8Fy8V8doxirNFcH", "mean_score": "0.44144144144144143", "stderr": "0.015718302238794238"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "kimi-k2-thinking", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.796581, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2-thinking-20251106", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "rows": [{"accuracy": "0.796581", "provider_name": "Google", "total_questions": "195"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "kimi-k2-thinking", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.695, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2-thinking-20251106", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-thinking-20251106", "rows": [{"accuracy": "0.695", "provider_name": "Google", "total_questions": "100"}, {"accuracy": "0.7233335", "provider_name": "Novita", "total_questions": "100"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen3-5-122b-a10b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-122b-a10b_none", "source_row": {"Best score (across scorers)": "0.17", "Country": "China", "Model version": "qwen3.5-122b-a10b_none", "Organization": "Alibaba", "Release date": "2026-02-24", "Started at": "2026-08-27T18:35:48.000Z", "id": "X5FL4gkJQYjzJ7axz23rfd", "mean_score": "0.17", "stderr": "0.03775251680686369"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "qwen3-5-122b-a10b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8073469346289753, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-122b-a10b-20260224", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "rows": [{"accuracy": "0.8073469346289753", "provider_name": "Alibaba", "total_questions": "2264"}, {"accuracy": "0.8427329166666666", "provider_name": "AtlasCloud", "total_questions": "2376"}, {"accuracy": "0.8440043168316831", "provider_name": "DeepInfra", "total_questions": "1515"}, {"accuracy": "0.8486092684824902", "provider_name": "Novita", "total_questions": "2313"}, {"accuracy": "0.8262167272727274", "provider_name": "SiliconFlow", "total_questions": "2178"}], "rule": "first-party provider (1 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-03", "model": "qwen3-5-122b-a10b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7471783002257337, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-122b-a10b-20260224", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-122b-a10b-20260224", "rows": [{"accuracy": "0.7471783002257337", "provider_name": "Alibaba", "total_questions": "443"}, {"accuracy": "0.7418181818181818", "provider_name": "AtlasCloud", "total_questions": "550"}, {"accuracy": "0.7579964141414142", "provider_name": "DeepInfra", "total_questions": "396"}, {"accuracy": "0.7596329412844037", "provider_name": "Novita", "total_questions": "545"}], "rule": "first-party provider (1 of 4 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "mimo-v2-6-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.758698, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.6-flash-20260921", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "rows": [{"accuracy": "0.7827002835443038", "provider_name": "DeepInfra", "total_questions": "790"}, {"accuracy": "0.7671156666666666", "provider_name": "GMICloud", "total_questions": "594"}, {"accuracy": "0.7626259999999999", "provider_name": "InferenceNet", "total_questions": "396"}, {"accuracy": "0.7485289698857738", "provider_name": "Io Net", "total_questions": "963"}, {"accuracy": "0.7160493333333333", "provider_name": "Makora", "total_questions": "594"}, {"accuracy": "0.7485157480916032", "provider_name": "Novita", "total_questions": "786"}, {"accuracy": "0.8521062280219781", "provider_name": "Venice", "total_questions": "364"}, {"accuracy": "0.758698", "provider_name": "Xiaomi", "total_questions": "594"}], "rule": "first-party provider (1 of 8 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "mimo-v2-6-flash", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.76, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "xiaomi/mimo-v2.6-flash-20260921", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "xiaomi/mimo-v2.6-flash-20260921", "rows": [{"accuracy": "0.771929874493927", "provider_name": "DeepInfra", "total_questions": "247"}, {"accuracy": "0.7633335", "provider_name": "GMICloud", "total_questions": "100"}, {"accuracy": "0.7766665", "provider_name": "InferenceNet", "total_questions": "100"}, {"accuracy": "0.7650601887550201", "provider_name": "Io Net", "total_questions": "249"}, {"accuracy": "0.7555553333333334", "provider_name": "Makora", "total_questions": "150"}, {"accuracy": "0.7833334999999999", "provider_name": "Novita", "total_questions": "200"}, {"accuracy": "0.716667", "provider_name": "Relace", "total_questions": "50"}, {"accuracy": "0.75", "provider_name": "Venice", "total_questions": "100"}, {"accuracy": "0.76", "provider_name": "Xiaomi", "total_questions": "150"}], "rule": "first-party provider (1 of 9 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "aider_polyglot", "eval_date": "2025-10-03", "model": "deepseek-v3-2-exp", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.742, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "DeepSeek-V3.2-Exp (Reasoner)", "source_row": {"command": "aider --model deepseek/deepseek-reasoner", "date": "2025-10-03", "dirname": "2025-10-03-09-45-34--deepseek-v3.2-reasoner", "model": "DeepSeek-V3.2-Exp (Reasoner)", "pass_rate_2": 74.2, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "deepseek-v3-2-exp", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.8265989999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v3.2-exp", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.2-exp", "rows": [{"accuracy": "0.83165", "provider_name": "AtlasCloud", "total_questions": "198"}, {"accuracy": "0.8265989999999999", "provider_name": "Novita", "total_questions": "198"}, {"accuracy": "0.814815", "provider_name": "SiliconFlow", "total_questions": "198"}], "rule": "median over providers (3 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "deepseek-v3-2-exp", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5672999999999999, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "DeepSeek-V3.2-Exp (Prompt + Thinking)", "source_row": {"Model": "DeepSeek-V3.2-Exp (Prompt + Thinking)", "Organization": "DeepSeek", "Overall Acc": "56.73%", "Rank": "14"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "deepseek-v3-2-exp", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.693333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v3.2-exp", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.2-exp", "rows": [{"accuracy": "0.74", "provider_name": "AtlasCloud", "total_questions": "50"}, {"accuracy": "0.693333", "provider_name": "SiliconFlow", "total_questions": "50"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-02-12", "model": "glm-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5", "source_row": {"Best score (across scorers)": "0.8", "Country": "China", "Model version": "glm-5", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-02-11", "Started at": "2026-02-12T16:08:22.578Z", "Training compute (FLOP)": "6.84e+24", "id": "ccbx7KQY9bTD563vzBXwSg", "mean_score": "0.8", "stderr": "0.06"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-15", "model": "glm-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7207792207792207, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5", "source_row": {"Best score (across scorers)": "0.7207792207792207", "Country": "China", "Model version": "glm-5", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-02-11", "Started at": "2026-02-15T15:40:10.848Z", "Training compute (FLOP)": "6.84e+24", "id": "hKkJEu4ueLiNWsTtrJ78PS", "mean_score": "0.7207792207792207", "stderr": "0.020894141602415234"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-02-12", "model": "glm-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8781725888324873, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-5", "source_row": {"Best score (across scorers)": "0.8781725888324873", "Country": "China", "Model version": "glm-5", "Organization": "Z.ai (Zhipu AI)", "Release date": "2026-02-11", "Started at": "2026-02-12T17:19:26.257Z", "Training compute (FLOP)": "6.84e+24", "id": "n4hyz75SVpwPEisKVVjvNg", "mean_score": "0.8781725888324873", "stderr": "0.02336331210092387"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "glm-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7285024173913045, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5-20260211", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-5-20260211", "rows": [{"accuracy": "0.7807238749999998", "provider_name": "Amazon Bedrock", "total_questions": "1584"}, {"accuracy": "0.7995168260869566", "provider_name": "AtlasCloud", "total_questions": "966"}, {"accuracy": "0.7573266413662239", "provider_name": "Baidu", "total_questions": "1581"}, {"accuracy": "0.5621250282776349", "provider_name": "DeepInfra", "total_questions": "389"}, {"accuracy": "0.422559", "provider_name": "DigitalOcean", "total_questions": "594"}, {"accuracy": "0.7091377451737451", "provider_name": "GMICloud", "total_questions": "1554"}, {"accuracy": "0.7391490737652341", "provider_name": "Novita", "total_questions": "1559"}, {"accuracy": "0.7466665885714285", "provider_name": "SiliconFlow", "total_questions": "1575"}, {"accuracy": "0.8084547777777777", "provider_name": "StreamLake", "total_questions": "1782"}, {"accuracy": "0.700785612121212", "provider_name": "Venice", "total_questions": "1485"}, {"accuracy": "0.7285024173913045", "provider_name": "Z.AI", "total_questions": "1380"}], "rule": "first-party provider (1 of 11 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "glm-5", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7422222222222222, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-5-20260211", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-5-20260211", "rows": [{"accuracy": "0.7674073333333333", "provider_name": "Amazon Bedrock", "total_questions": "450"}, {"accuracy": "0.7722220000000001", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.7555555555555555", "provider_name": "Baidu", "total_questions": "450"}, {"accuracy": "0.74416675", "provider_name": "DigitalOcean", "total_questions": "200"}, {"accuracy": "0.728889", "provider_name": "GMICloud", "total_questions": "450"}, {"accuracy": "0.7496295555555555", "provider_name": "Novita", "total_questions": "450"}, {"accuracy": "0.7942177959183673", "provider_name": "Parasail", "total_questions": "98"}, {"accuracy": "0.751111", "provider_name": "SiliconFlow", "total_questions": "450"}, {"accuracy": "0.77", "provider_name": "StreamLake", "total_questions": "500"}, {"accuracy": "0.747037111111111", "provider_name": "Venice", "total_questions": "450"}, {"accuracy": "0.7422222222222222", "provider_name": "Z.AI", "total_questions": "450"}], "rule": "first-party provider (1 of 11 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-30", "model": "gemini-3-1-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2771929824561403, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-flash-lite_high", "source_row": {"Best score (across scorers)": "0.2771929824561403", "Country": "United States of America", "Model version": "gemini-3.1-flash-lite_high", "Organization": "Google", "Release date": "2026-03-03", "Started at": "2026-08-30T22:34:53.000Z", "id": "MVZf59jt9uEoWG9w5W9apB", "mean_score": "0.2771929824561403", "stderr": "0.026560920735701255"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-06", "model": "gemini-3-1-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-flash-lite_high", "source_row": {"Best score (across scorers)": "0.8", "Country": "United States of America", "Model version": "gemini-3.1-flash-lite_high", "Organization": "Google", "Release date": "2026-03-03", "Started at": "2026-08-06T22:29:57.000Z", "id": "NMdaAXyqqPRmWfuxZUfHFF", "mean_score": "0.8", "stderr": "0.06030226891555273"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "gemini-3-1-flash-lite", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8181818181818182, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-3.1-flash-lite_high", "source_row": {"Best score (across scorers)": "0.8181818181818182", "Country": "United States of America", "Model version": "gemini-3.1-flash-lite_high", "Organization": "Google", "Release date": "2026-03-03", "Started at": "2026-08-06T22:26:07.000Z", "id": "nE4hmJo3QV6X5jeGWAeAze", "mean_score": "0.8181818181818182", "stderr": "0.027479603010538825"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "qwen-3-5-plus-hosted-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-plus", "source_row": {"Best score (across scorers)": "0.08", "Country": "China", "Model version": "qwen3.5-plus", "Organization": "Alibaba", "Release date": "2026-02-16", "Started at": "2026-10-13T13:39:53.000Z", "id": "XEaDaqM3oaPcrtQ6XH5fGS", "mean_score": "0.08", "stderr": "0.027265992434429072"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen-3-5-plus-hosted-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-plus_none", "source_row": {"Best score (across scorers)": "0.17", "Country": "China", "Model version": "qwen3.5-plus_none", "Organization": "Alibaba", "Release date": "2026-02-16", "Started at": "2026-08-27T18:23:25.000Z", "id": "XbtWdvRz65dbPPnHcSBvmF", "mean_score": "0.17", "stderr": "0.03775251680686369"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen-3-5-plus-hosted-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-plus", "source_row": {"Best score (across scorers)": "0.8666666666666667", "Country": "China", "Model version": "qwen3.5-plus", "Organization": "Alibaba", "Release date": "2026-02-16", "Started at": "2026-08-07T00:11:58.000Z", "id": "cFbntXuxUtfreULvt7vEt7", "mean_score": "0.8666666666666667", "stderr": "0.051247074319053836"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen-3-5-plus-hosted-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8484848484848485, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-plus", "source_row": {"Best score (across scorers)": "0.8484848484848485", "Country": "China", "Model version": "qwen3.5-plus", "Organization": "Alibaba", "Release date": "2026-02-16", "Started at": "2026-08-07T00:20:32.000Z", "id": "mGXYSUNCMisvaRzhQp6g2L", "mean_score": "0.8484848484848485", "stderr": "0.025545650426603637"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen-3-5-plus-hosted-397b-a17b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2535070140280561, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-plus", "source_row": {"Best score (across scorers)": "0.2535070140280561", "Country": "China", "Model version": "qwen3.5-plus", "Organization": "Alibaba", "Release date": "2026-02-16", "Started at": "2026-08-27T21:29:42.000Z", "id": "jrawnM2wHFbVfJFJsap4kS", "mean_score": "0.2535070140280561", "stderr": "0.013777178222916077"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-05-07", "model": "gemini-2-5-pro-mar-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9556268882175226, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro-preview-03-25", "source_row": {"Best score (across scorers)": "0.9556268882175226", "Country": "United States of America", "Model version": "gemini-2.5-pro-preview-03-25", "Organization": "Google DeepMind", "Release date": "2025-03-31", "Started at": "2025-05-07T21:59:53.901Z", "id": "XRkZCzSoQC5yYiQ3xitNdC", "mean_score": "0.9556268882175226", "stderr": "0.004475169260814502"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-12", "model": "gemini-2-5-pro-mar-2025", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.7290000000000001, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Gemini 2.5 Pro Preview 03-25", "source_row": {"command": "aider --model gemini/gemini-2.5-pro-preview-03-25", "date": "2025-04-12", "dirname": "2025-04-12-04-55-50--gemini-25-pro-diff-fenced", "model": "Gemini 2.5 Pro Preview 03-25", "pass_rate_2": 72.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-03-31", "model": "gemini-2-5-pro-mar-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8383838383838383, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro-exp-03-25", "source_row": {"Best score (across scorers)": "0.8383838383838383", "Country": "United States of America", "Model version": "gemini-2.5-pro-exp-03-25", "Organization": "Google DeepMind", "Release date": "2025-03-25", "Started at": "2025-03-31T16:06:24.669Z", "id": "WKjzEQdfnBHnYeZoKeCWnB", "mean_score": "0.8383838383838383", "stderr": "0.02622591986362927"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "qwen-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-flash", "source_row": {"Best score (across scorers)": "0.12", "Country": "China", "Model version": "qwen3.6-flash", "Organization": "Alibaba", "Release date": "2026-04-27", "Started at": "2026-10-13T13:33:28.000Z", "id": "j99TDv5LAYUSvTUcR47T75", "mean_score": "0.12", "stderr": "0.032659863237109045"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.18, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-flash_none", "source_row": {"Best score (across scorers)": "0.18", "Country": "China", "Model version": "qwen3.6-flash_none", "Organization": "Alibaba", "Release date": "2026-04-27", "Started at": "2026-08-27T18:14:08.000Z", "id": "3QtkFWzSLzd4hgYGsNd5cL", "mean_score": "0.18", "stderr": "0.03861229196653691"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-29", "model": "qwen-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22456140350877193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-flash_none", "source_row": {"Best score (across scorers)": "0.22456140350877193", "Country": "China", "Model version": "qwen3.6-flash_none", "Organization": "Alibaba", "Release date": "2026-04-27", "Started at": "2026-08-29T11:06:46.000Z", "id": "RoscWx4NUGVqcHdko6QhKd", "mean_score": "0.22456140350877193", "stderr": "0.024761793289496806"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8444444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-flash", "source_row": {"Best score (across scorers)": "0.8444444444444444", "Country": "China", "Model version": "qwen3.6-flash", "Organization": "Alibaba", "Release date": "2026-04-27", "Started at": "2026-08-07T00:02:08.000Z", "id": "9zx2EXPTvPmxsL9qFCjP3x", "mean_score": "0.8444444444444444", "stderr": "0.05463890236888291"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8333333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-flash", "source_row": {"Best score (across scorers)": "0.8333333333333334", "Country": "China", "Model version": "qwen3.6-flash", "Organization": "Alibaba", "Release date": "2026-04-27", "Started at": "2026-08-07T00:11:57.000Z", "id": "H2FvRXagQRQmiNa7WrNMU2", "mean_score": "0.8333333333333334", "stderr": "0.026552207828215258"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen-3-6-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.15915915915915915, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.6-flash", "source_row": {"Best score (across scorers)": "0.15915915915915915", "Country": "China", "Model version": "qwen3.6-flash", "Organization": "Alibaba", "Release date": "2026-04-27", "Started at": "2026-08-27T19:33:42.000Z", "id": "FoMSS4FmH3VxpsvZiUJPCv", "mean_score": "0.15915915915915915", "stderr": "0.011579970321213651"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-28", "model": "qwen3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.15, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-flash", "source_row": {"Best score (across scorers)": "0.15", "Country": "China", "Model version": "qwen3.7-flash", "Organization": "Alibaba", "Release date": "2026-07-27", "Started at": "2026-08-28T07:12:24.000Z", "id": "5e4Jp9dxtNisWx2wbRRZxM", "mean_score": "0.15", "stderr": "0.03588702812826367"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-29", "model": "qwen3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.19298245614035087, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-flash_none", "source_row": {"Best score (across scorers)": "0.19298245614035087", "Country": "China", "Model version": "qwen3.7-flash_none", "Organization": "Alibaba", "Release date": "2026-07-27", "Started at": "2026-08-29T11:07:51.000Z", "id": "doNrgGQY5SytQV22WRqw5L", "mean_score": "0.19298245614035087", "stderr": "0.02341753795065531"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-flash", "source_row": {"Best score (across scorers)": "0.8666666666666667", "Country": "China", "Model version": "qwen3.7-flash", "Organization": "Alibaba", "Release date": "2026-07-27", "Started at": "2026-08-07T00:02:46.000Z", "id": "TjoS7sNuqVTc4LVyGRo4wa", "mean_score": "0.8666666666666667", "stderr": "0.051247074319053836"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen3-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8232323232323232, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.7-flash", "source_row": {"Best score (across scorers)": "0.8232323232323232", "Country": "China", "Model version": "qwen3.7-flash", "Organization": "Alibaba", "Release date": "2026-07-27", "Started at": "2026-08-07T00:03:33.000Z", "id": "i6i3gQf3ng2aAipvowPpws", "mean_score": "0.8232323232323232", "stderr": "0.027178752639044908"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-08-02", "model": "gpt-5-5-instant", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.024390243902439025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-instant", "source_row": {"Best score (across scorers)": "0.024390243902439025", "Country": "United States of America", "Model version": "gpt-5.5-instant", "Organization": "OpenAI", "Release date": "2026-05-05", "Started at": "2026-08-02T03:23:36.000Z", "id": "2UmuBfhBvYbbnTg5BnQqov", "mean_score": "0.024390243902439025", "stderr": "0.024390243902439022"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-02", "model": "gpt-5-5-instant", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2631578947368421, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-instant", "source_row": {"Best score (across scorers)": "0.2631578947368421", "Country": "United States of America", "Model version": "gpt-5.5-instant", "Organization": "OpenAI", "Release date": "2026-05-05", "Started at": "2026-08-02T03:23:36.000Z", "id": "gGcHjE6ppy54ahN2pfxsGH", "mean_score": "0.2631578947368421", "stderr": "0.026129809832490088"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-02", "model": "gpt-5-5-instant", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6805555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-instant", "source_row": {"Best score (across scorers)": "0.6805555555555556", "Country": "United States of America", "Model version": "gpt-5.5-instant", "Organization": "OpenAI", "Release date": "2026-05-05", "Started at": "2026-08-02T02:46:37.000Z", "id": "hGzLQWbxPdgQiiMbaDqF8k", "mean_score": "0.6805555555555556", "stderr": "0.062127114361967634"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-02", "model": "gpt-5-5-instant", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8251262626262627, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-5.5-instant", "source_row": {"Best score (across scorers)": "0.8251262626262627", "Country": "United States of America", "Model version": "gpt-5.5-instant", "Organization": "OpenAI", "Release date": "2026-05-05", "Started at": "2026-08-02T02:46:37.000Z", "id": "WPVhgi8NT9grAT7kfipT9G", "mean_score": "0.8251262626262627", "stderr": "0.022532782662252126"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "qwen-3-5-flash-hosted-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-flash", "source_row": {"Best score (across scorers)": "0.14", "Country": "China", "Model version": "qwen3.5-flash", "Organization": "Alibaba", "Release date": "2026-02-25", "Started at": "2026-10-13T13:39:44.000Z", "id": "fEgRyzyFYZraNnkkbRVSSg", "mean_score": "0.14", "stderr": "0.034873508801977704"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen-3-5-flash-hosted-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-flash", "source_row": {"Best score (across scorers)": "0.2", "Country": "China", "Model version": "qwen3.5-flash", "Organization": "Alibaba", "Release date": "2026-02-25", "Started at": "2026-08-27T18:27:45.000Z", "id": "b9csPZ2c9CAsRXmzzfVoEW", "mean_score": "0.2", "stderr": "0.04020151261036849"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-28", "model": "qwen-3-5-flash-hosted-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1824561403508772, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-flash_none", "source_row": {"Best score (across scorers)": "0.1824561403508772", "Country": "China", "Model version": "qwen3.5-flash_none", "Organization": "Alibaba", "Release date": "2026-02-25", "Started at": "2026-08-28T12:28:39.000Z", "id": "afxoicJc4C6caVDwUpsiME", "mean_score": "0.1824561403508772", "stderr": "0.022917941277334045"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "qwen-3-5-flash-hosted-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8444444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-flash", "source_row": {"Best score (across scorers)": "0.8444444444444444", "Country": "China", "Model version": "qwen3.5-flash", "Organization": "Alibaba", "Release date": "2026-02-25", "Started at": "2026-08-07T00:00:21.000Z", "id": "awDKyKbPXRzBVZzfdQucef", "mean_score": "0.8444444444444444", "stderr": "0.05463890236888291"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-07", "model": "qwen-3-5-flash-hosted-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8232323232323232, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-flash", "source_row": {"Best score (across scorers)": "0.8232323232323232", "Country": "China", "Model version": "qwen3.5-flash", "Organization": "Alibaba", "Release date": "2026-02-25", "Started at": "2026-08-07T00:00:59.000Z", "id": "S7wbot7JR53BAjM8F9gaws", "mean_score": "0.8232323232323232", "stderr": "0.027178752639044908"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen-3-5-flash-hosted-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2032032032032032, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-flash", "source_row": {"Best score (across scorers)": "0.2032032032032032", "Country": "China", "Model version": "qwen3.5-flash", "Organization": "Alibaba", "Release date": "2026-02-25", "Started at": "2026-08-27T19:32:07.000Z", "id": "K8zTckSXLKLesoLBv24TWc", "mean_score": "0.2032032032032032", "stderr": "0.012737194890497593"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "qwen3-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.05, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-max-2025-09-23", "source_row": {"Best score (across scorers)": "0.05", "Country": "China", "Model version": "qwen3-max-2025-09-23", "Organization": "Alibaba", "Release date": "2025-09-24", "Started at": "2026-08-27T18:07:41.000Z", "Training compute (FLOP)": "1.512e+25", "id": "SfzmmHYZjuvWnvoYWit58g", "mean_score": "0.05", "stderr": "0.021904291355759033"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-30", "model": "qwen3-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.18947368421052632, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-max-2025-09-23", "source_row": {"Best score (across scorers)": "0.18947368421052632", "Country": "China", "Model version": "qwen3-max-2025-09-23", "Organization": "Alibaba", "Release date": "2025-09-24", "Started at": "2026-08-30T22:34:08.000Z", "Training compute (FLOP)": "1.512e+25", "id": "NrTH3EYHjp4hPdxgpCPGTX", "mean_score": "0.18947368421052632", "stderr": "0.023254062737737245"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-10-09", "model": "qwen3-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9712990936555891, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-max-2025-09-23", "source_row": {"Best score (across scorers)": "0.9712990936555891", "Country": "China", "Model version": "qwen3-max-2025-09-23", "Organization": "Alibaba", "Release date": "2025-09-24", "Started at": "2025-10-09T15:30:36.002Z", "Training compute (FLOP)": "1.512e+25", "id": "f2vxirYMpXwkaZxb4ywnYM", "mean_score": "0.9712990936555891", "stderr": "0.004200694815744905"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-06", "model": "qwen3-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-max-2025-09-23", "source_row": {"Best score (across scorers)": "0.7333333333333333", "Country": "China", "Model version": "qwen3-max-2025-09-23", "Organization": "Alibaba", "Release date": "2025-09-24", "Started at": "2025-10-06T14:32:01.826Z", "Training compute (FLOP)": "1.512e+25", "id": "ULeuPtwp3ZoyuXQYuuQ82i", "mean_score": "0.7333333333333333", "stderr": "0.059139451700829515"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-10-06", "model": "qwen3-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.726010101010101, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-max-2025-09-23", "source_row": {"Best score (across scorers)": "0.726010101010101", "Country": "China", "Model version": "qwen3-max-2025-09-23", "Organization": "Alibaba", "Release date": "2025-09-24", "Started at": "2025-10-06T16:27:30.400Z", "Training compute (FLOP)": "1.512e+25", "id": "46RCssznDADYeg8MuSgvsj", "mean_score": "0.726010101010101", "stderr": "0.02734528103171235"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "qwen3-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4874874874874875, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-max-2025-09-23", "source_row": {"Best score (across scorers)": "0.4874874874874875", "Country": "China", "Model version": "qwen3-max-2025-09-23", "Organization": "Alibaba", "Release date": "2025-09-24", "Started at": "2026-08-27T19:32:02.000Z", "Training compute (FLOP)": "1.512e+25", "id": "fXG2GPt5F25tHRTPhAoZ4K", "mean_score": "0.4874874874874875", "stderr": "0.01582226675546796"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "gemma-4-26b-a4b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8222222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-4-26b-a4b-it_minimal", "source_row": {"Best score (across scorers)": "0.8222222222222222", "Country": "United States of America", "Model version": "gemma-4-26b-a4b-it_minimal", "Organization": "Google DeepMind", "Release date": "2026-04-02", "Started at": "2026-08-28T12:55:56.000Z", "id": "mstwSH7noJfvhGcd6Lz5kd", "mean_score": "0.8222222222222222", "stderr": "0.05763774795025092"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "gemma-4-26b-a4b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7323232323232324, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-4-26b-a4b-it_minimal", "source_row": {"Best score (across scorers)": "0.7323232323232324", "Country": "United States of America", "Model version": "gemma-4-26b-a4b-it_minimal", "Organization": "Google DeepMind", "Release date": "2026-04-02", "Started at": "2026-08-06T23:59:25.000Z", "id": "Usx9KaiXvHZU4qrzgc4sLg", "mean_score": "0.7323232323232324", "stderr": "0.031544498882702825"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "gemma-4-26b-a4b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7762077565217391, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemma-4-26b-a4b-it-20260403", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "rows": [{"accuracy": "0.7782009841269842", "provider_name": "Cloudflare", "total_questions": "1575"}, {"accuracy": "0.7659312647058824", "provider_name": "CoreWeave", "total_questions": "544"}, {"accuracy": "0.6021274927659573", "provider_name": "Darkbloom", "total_questions": "1175"}, {"accuracy": "0.6747943299556681", "provider_name": "DeepInfra", "total_questions": "1579"}, {"accuracy": "0.7622055000000001", "provider_name": "DekaLLM", "total_questions": "792"}, {"accuracy": "0.7762077565217391", "provider_name": "Google", "total_questions": "1380"}, {"accuracy": "0.7886248285229202", "provider_name": "Io Net", "total_questions": "589"}, {"accuracy": "0.46128619685039374", "provider_name": "Ionstream", "total_questions": "254"}, {"accuracy": "0.7457915", "provider_name": "Makora", "total_questions": "396"}, {"accuracy": "0.7485971111111113", "provider_name": "NextBit", "total_questions": "1782"}, {"accuracy": "0.733048471794872", "provider_name": "Novita", "total_questions": "1170"}, {"accuracy": "0.7619047142857143", "provider_name": "Parasail", "total_questions": "1386"}, {"accuracy": "0.7741301666666666", "provider_name": "SiliconFlow", "total_questions": "1188"}, {"accuracy": "0.31216928571428576", "provider_name": "Venice", "total_questions": "1386"}], "rule": "first-party provider (1 of 14 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "gemma-4-26b-a4b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.5733334999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemma-4-26b-a4b-it-20260403", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-26b-a4b-it-20260403", "rows": [{"accuracy": "0.6666667246963563", "provider_name": "Cloudflare", "total_questions": "494"}, {"accuracy": "0.6521739130434783", "provider_name": "Darkbloom", "total_questions": "299"}, {"accuracy": "0.6958144439461884", "provider_name": "DeepInfra", "total_questions": "446"}, {"accuracy": "0.6866665000000001", "provider_name": "DekaLLM", "total_questions": "200"}, {"accuracy": "0.5733334999999999", "provider_name": "Google", "total_questions": "400"}, {"accuracy": "0.6913265306122449", "provider_name": "Io Net", "total_questions": "196"}, {"accuracy": "0.693333", "provider_name": "Ionstream", "total_questions": "50"}, {"accuracy": "0.735", "provider_name": "Makora", "total_questions": "100"}, {"accuracy": "0.682", "provider_name": "NextBit", "total_questions": "500"}, {"accuracy": "0.6917516734693877", "provider_name": "Novita", "total_questions": "392"}, {"accuracy": "0.6570882643678161", "provider_name": "Venice", "total_questions": "348"}], "rule": "first-party provider (1 of 11 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "qwen3-5-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-35b-a3b_none", "source_row": {"Best score (across scorers)": "0.7", "Country": "China", "Model version": "qwen3.5-35b-a3b_none", "Organization": "Alibaba", "Release date": "2026-02-24", "Started at": "2026-08-30T22:45:56.000Z", "id": "h9aoDxyWEhybvARHpyuoco", "mean_score": "0.7", "stderr": "0.06003996985174109"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "qwen3-5-35b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8345959595959596, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-35b-a3b", "source_row": {"Best score (across scorers)": "0.8345959595959596", "Country": "China", "Model version": "qwen3.5-35b-a3b", "Organization": "Alibaba", "Release date": "2026-02-24", "Started at": "2026-08-28T18:38:38.000Z", "id": "RWjuCuVGvaRd4vDjJJ6aP2", "mean_score": "0.8345959595959596", "stderr": "0.02179018174848238"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "qwen3-5-35b-a3b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7500000000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-35b-a3b-20260224", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "rows": [{"accuracy": "0.26430966666666666", "provider_name": "AkashML", "total_questions": "594"}, {"accuracy": "0.7500000000000001", "provider_name": "Alibaba", "total_questions": "1584"}, {"accuracy": "0.8124998749999999", "provider_name": "AtlasCloud", "total_questions": "1584"}, {"accuracy": "0.8258873707224335", "provider_name": "CoreWeave", "total_questions": "1052"}, {"accuracy": "0.7845117999999999", "provider_name": "Darkbloom", "total_questions": "990"}, {"accuracy": "0.8249519000000001", "provider_name": "DeepInfra", "total_questions": "1730"}, {"accuracy": "0.7805356401045069", "provider_name": "Parasail", "total_questions": "1531"}, {"accuracy": "0.8234428749999999", "provider_name": "SiliconFlow", "total_questions": "1584"}, {"accuracy": "0.8156565", "provider_name": "Venice", "total_questions": "1584"}], "rule": "first-party provider (1 of 9 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "qwen3-5-35b-a3b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7351189375, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-35b-a3b-20260224", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-35b-a3b-20260224", "rows": [{"accuracy": "0.58166675", "provider_name": "AkashML", "total_questions": "200"}, {"accuracy": "0.7351189375", "provider_name": "Alibaba", "total_questions": "448"}, {"accuracy": "0.6596296666666667", "provider_name": "AtlasCloud", "total_questions": "450"}, {"accuracy": "0.7155646528925619", "provider_name": "CoreWeave", "total_questions": "242"}, {"accuracy": "0.7693334", "provider_name": "Darkbloom", "total_questions": "250"}, {"accuracy": "0.6734496104651163", "provider_name": "Parasail", "total_questions": "344"}, {"accuracy": "0.6933333333333334", "provider_name": "Venice", "total_questions": "450"}], "rule": "first-party provider (1 of 7 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-07-25", "model": "claude-opus-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-1-20250805_24K", "source_row": {"Best score (across scorers)": "0.21", "Country": "United States of America", "Model version": "claude-opus-4-1-20250805_24K", "Organization": "Anthropic", "Release date": "2025-08-05", "Started at": "2026-07-25T07:05:43.000Z", "id": "2AbeCjy9w4KJWKhbKjtFTJ", "mean_score": "0.21", "stderr": "0.040936018074033236"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t4", "eval_date": "2026-06-11", "model": "claude-opus-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.024390243902439025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-1-20250805_32K", "source_row": {"Best score (across scorers)": "0.024390243902439025", "Country": "United States of America", "Model version": "claude-opus-4-1-20250805_32K", "Organization": "Anthropic", "Release date": "2025-08-05", "Started at": "2026-06-11T21:15:07.000Z", "id": "9zT72xRR6wZhiwraJTNPyz", "mean_score": "0.024390243902439025", "stderr": "0.024390243902439022"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tier_4_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-06-11", "model": "claude-opus-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12631578947368421, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-1-20250805_32K", "source_row": {"Best score (across scorers)": "0.12631578947368421", "Country": "United States of America", "Model version": "claude-opus-4-1-20250805_32K", "Organization": "Anthropic", "Release date": "2025-08-05", "Started at": "2026-06-11T21:15:07.000Z", "id": "5eRkmXCURH2pRbhJoukfnC", "mean_score": "0.12631578947368421", "stderr": "0.0197127354633578"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-08-05", "model": "claude-opus-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6888888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-1-20250805_27K", "source_row": {"Best score (across scorers)": "0.6888888888888889", "Country": "United States of America", "Model version": "claude-opus-4-1-20250805_27K", "Organization": "Anthropic", "Release date": "2025-08-05", "Started at": "2025-08-05T16:53:30.846Z", "id": "UH9RY8MCAotDgFocHKA4j6", "mean_score": "0.6888888888888889", "stderr": "0.06979205927323111"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "swe_bench_verified", "eval_date": "2026-02-11", "model": "claude-opus-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7334710743801653, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-1-20250805", "source_row": {"Best score (across scorers)": "0.7334710743801653", "Country": "United States of America", "Model version": "claude-opus-4-1-20250805", "Organization": "Anthropic", "Release date": "2025-08-05", "Started at": "2026-02-11T21:33:53.502Z", "id": "cA6Mq67yaynLhY8F5kes3c", "mean_score": "0.7334710743801653", "stderr": "0.020118245554677085"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#swe_bench_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-08-05", "model": "claude-opus-4-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7727272727272727, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-opus-4-1-20250805_16K", "source_row": {"Best score (across scorers)": "0.7727272727272727", "Country": "United States of America", "Model version": "claude-opus-4-1-20250805_16K", "Organization": "Anthropic", "Release date": "2025-08-05", "Started at": "2025-08-05T16:51:47.698Z", "id": "erLgyKQs2mg2memE9SJvK5", "mean_score": "0.7727272727272727", "stderr": "0.029857515673386438"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "kimi-k2-sep-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7474749999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2-0905", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "moonshotai/kimi-k2-0905", "rows": [{"accuracy": "0.7474749999999999", "provider_name": "Groq", "total_questions": "198"}, {"accuracy": "0.808081", "provider_name": "Novita", "total_questions": "198"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "kimi-k2-sep-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6366665, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "moonshotai/kimi-k2-0905", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "moonshotai/kimi-k2-0905", "rows": [{"accuracy": "0.6366665", "provider_name": "Novita", "total_questions": "100"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-05-29", "model": "deepseek-r1-may-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9663897280966768, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-0528", "source_row": {"Best score (across scorers)": "0.9663897280966768", "Country": "China", "Model version": "DeepSeek-R1-0528", "Organization": "DeepSeek", "Release date": "2025-05-28", "Started at": "2025-05-29T14:09:30.970Z", "Training compute (FLOP)": "4.020010000000001e+24", "id": "2wd9khVKiBzsAMUquz2R5s", "mean_score": "0.9663897280966768", "stderr": "0.0042156161735636515"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-05-29", "model": "deepseek-r1-may-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6638888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-0528", "source_row": {"Best score (across scorers)": "0.6638888888888889", "Country": "China", "Model version": "DeepSeek-R1-0528", "Organization": "DeepSeek", "Release date": "2025-05-28", "Started at": "2025-05-29T10:59:10.779Z", "Training compute (FLOP)": "4.020010000000001e+24", "id": "bBV5wCrrZtihrgxEsg9jPD", "mean_score": "0.6638888888888889", "stderr": "0.04911177955575991"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-05-29", "model": "deepseek-r1-may-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7632575757575758, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-0528", "source_row": {"Best score (across scorers)": "0.7632575757575758", "Country": "China", "Model version": "DeepSeek-R1-0528", "Organization": "DeepSeek", "Release date": "2025-05-28", "Started at": "2025-05-29T13:30:43.463Z", "Training compute (FLOP)": "4.020010000000001e+24", "id": "LiRiryJsGb3h5UuxX8bmJv", "mean_score": "0.7632575757575758", "stderr": "0.024023818843083005"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "deepseek-r1-may-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.7851099187817259, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-r1-0528", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-r1-0528", "rows": [{"accuracy": "0.7902960850403475", "provider_name": "DeepInfra", "total_questions": "1611"}, {"accuracy": "0.651515", "provider_name": "Novita", "total_questions": "594"}, {"accuracy": "0.7964659214659687", "provider_name": "SiliconFlow", "total_questions": "1528"}, {"accuracy": "0.7851099187817259", "provider_name": "StreamLake", "total_questions": "1773"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "deepseek-r1-may-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.47874697986577186, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-r1-0528", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-r1-0528", "rows": [{"accuracy": "0.5789978786127168", "provider_name": "Novita", "total_questions": "346"}, {"accuracy": "0.47874697986577186", "provider_name": "SiliconFlow", "total_questions": "298"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-05-08", "model": "gemini-2-5-pro-may-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9590256797583081, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro-preview-05-06", "source_row": {"Best score (across scorers)": "0.9590256797583081", "Country": "United States of America", "Model version": "gemini-2.5-pro-preview-05-06", "Organization": "Google DeepMind", "Release date": "2025-05-06", "Started at": "2025-05-08T05:28:17.657Z", "id": "Mi8LCxmduKukxazAtmuhqg", "mean_score": "0.9590256797583081", "stderr": "0.004221388011004008"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-05-07", "model": "gemini-2-5-pro-may-2025", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.769, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Gemini 2.5 Pro Preview 05-06", "source_row": {"command": "aider --model gemini/gemini-2.5-pro-preview-05-06", "date": "2025-05-07", "dirname": "2025-05-07-19-32-40--gemini0506-diff-fenced-completion_cost", "model": "Gemini 2.5 Pro Preview 05-06", "pass_rate_2": 76.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-06-03", "model": "gemini-2-5-pro-may-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6666666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-pro-preview-05-06", "source_row": {"Best score (across scorers)": "0.6666666666666666", "Country": "United States of America", "Model version": "gemini-2.5-pro-preview-05-06", "Organization": "Google DeepMind", "Release date": "2025-05-06", "Started at": "2025-06-03T09:10:11.019Z", "id": "TT66gGiVoeYLGQMCESPrEG", "mean_score": "0.6666666666666666", "stderr": "0.033586181457325226"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-06", "model": "gemma-4-31b-it", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-4-31b-it_minimal", "source_row": {"Best score (across scorers)": "0.7333333333333333", "Country": "United States of America", "Model version": "gemma-4-31b-it_minimal", "Organization": "Google DeepMind", "Release date": "2026-04-02", "Started at": "2026-08-06T23:59:45.000Z", "id": "jFJucVunfLgoQqoFvPVQWP", "mean_score": "0.7333333333333333", "stderr": "0.06666666666666667"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "gemma-4-31b-it", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7575757575757576, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-4-31b-it_minimal", "source_row": {"Best score (across scorers)": "0.7575757575757576", "Country": "United States of America", "Model version": "gemma-4-31b-it_minimal", "Organization": "Google DeepMind", "Release date": "2026-04-02", "Started at": "2026-08-06T23:59:38.000Z", "id": "P5L8qbXo42vTvbU3UvTNjk", "mean_score": "0.7575757575757576", "stderr": "0.03053289223393202"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "gemma-4-31b-it", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.81649825, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemma-4-31b-it-20260402", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-4-31b-it-20260402", "rows": [{"accuracy": "0.7659344612403101", "provider_name": "Cerebras", "run_count": "4", "total_questions": "774"}, {"accuracy": "0.81649825", "provider_name": "CoreWeave", "run_count": "8", "total_questions": "1584"}, {"accuracy": "0.8165709598662209", "provider_name": "DeepInfra", "run_count": "22", "total_questions": "4186"}, {"accuracy": "0.7154881249999999", "provider_name": "Friendli", "run_count": "8", "total_questions": "1584"}, {"accuracy": "0.8357119447513812", "provider_name": "ModelRun", "run_count": "11", "total_questions": "2172"}, {"accuracy": "0.6662457499999999", "provider_name": "OpenInference", "run_count": "4", "total_questions": "792"}, {"accuracy": "0.8466919554707381", "provider_name": "Parasail", "run_count": "8", "total_questions": "1572"}, {"accuracy": "0.8173077377622378", "provider_name": "Phala", "run_count": "3", "total_questions": "572"}, {"accuracy": "0.8504087142857142", "provider_name": "SiliconFlow", "run_count": "7", "total_questions": "1386"}, {"accuracy": "0.6652237142857144", "provider_name": "Venice", "run_count": "7", "total_questions": "1386"}], "rule": "median over providers (10 of 10 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-27", "model": "gemma-4-31b-it", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.104, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-4-31b-it", "source_row": {"Best score (across scorers)": "0.104", "Country": "United States of America", "Model version": "gemma-4-31b-it", "Organization": "Google DeepMind", "Release date": "2026-04-02", "Started at": "2026-08-27T19:30:07.000Z", "id": "D5cbXacPnJ8akq2jAtifEQ", "mean_score": "0.104", "stderr": "0.009658016218524242"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "gemma-4-31b-it", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.756666625, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemma-4-31b-it-20260402", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-4-31b-it-20260402", "rows": [{"accuracy": "0.5648772953020134", "provider_name": "Cerebras", "total_questions": "149"}, {"accuracy": "0.3980540203045685", "provider_name": "Chutes", "total_questions": "394"}, {"accuracy": "0.7586728009049774", "provider_name": "CoreWeave", "total_questions": "442"}, {"accuracy": "0.7859532642140468", "provider_name": "Crusoe", "total_questions": "299"}, {"accuracy": "0.775245761394102", "provider_name": "DeepInfra", "total_questions": "746"}, {"accuracy": "0.7744445", "provider_name": "Friendli", "total_questions": "300"}, {"accuracy": "0.7624549738058551", "provider_name": "ModelRun", "total_questions": "649"}, {"accuracy": "0.7533333333333333", "provider_name": "Morph", "total_questions": "150"}, {"accuracy": "0.6962752034383954", "provider_name": "Novita", "total_questions": "349"}, {"accuracy": "0.765", "provider_name": "OpenInference", "total_questions": "300"}, {"accuracy": "0.38583349999999994", "provider_name": "Parasail", "total_questions": "400"}, {"accuracy": "0.7366665", "provider_name": "Phala", "total_questions": "100"}, {"accuracy": "0.765625", "provider_name": "SambaNova", "total_questions": "96"}, {"accuracy": "0.584762", "provider_name": "SiliconFlow", "total_questions": "350"}, {"accuracy": "0.756666625", "provider_name": "Venice", "total_questions": "400"}], "rule": "median over providers (15 of 15 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-03", "model": "qwen3-coder-next", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7575756666666668, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-coder-next-2025-02-03", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "rows": [{"accuracy": "0.7575756666666668", "provider_name": "Alibaba", "total_questions": "1188"}, {"accuracy": "0.7308918662420382", "provider_name": "Ionstream", "total_questions": "314"}, {"accuracy": "0.7404603333333333", "provider_name": "Novita", "total_questions": "1188"}, {"accuracy": "0.7435209222520109", "provider_name": "Parasail", "total_questions": "1119"}, {"accuracy": "0.7626260000000001", "provider_name": "PrimeIntellect", "total_questions": "198"}, {"accuracy": "0.7493987142857143", "provider_name": "StreamLake", "total_questions": "1386"}], "rule": "first-party provider (1 of 6 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-03", "model": "qwen3-coder-next", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.5762463812316716, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-coder-next-2025-02-03", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-next-2025-02-03", "rows": [{"accuracy": "0.5762463812316716", "provider_name": "Alibaba", "total_questions": "341"}, {"accuracy": "0.5902778229166666", "provider_name": "Ionstream", "total_questions": "96"}, {"accuracy": "0.22095242857142855", "provider_name": "Novita", "total_questions": "350"}, {"accuracy": "0.5943487528735633", "provider_name": "Parasail", "total_questions": "348"}, {"accuracy": "0.186667", "provider_name": "PrimeIntellect", "total_questions": "50"}, {"accuracy": "0.59958325", "provider_name": "StreamLake", "total_questions": "400"}], "rule": "first-party provider (1 of 6 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "o1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-2024-12-17_high", "source_row": {"Best score (across scorers)": "0.1", "Country": "United States of America", "Model version": "o1-2024-12-17_high", "Organization": "OpenAI", "Release date": "2024-12-17", "Started at": "2026-10-13T14:12:19.000Z", "id": "UtRhB562A5f4SuFsg5MPrJ", "mean_score": "0.1", "stderr": "0.03015113445777636"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-28", "model": "o1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14736842105263157, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-2024-12-17_high", "source_row": {"Best score (across scorers)": "0.14736842105263157", "Country": "United States of America", "Model version": "o1-2024-12-17_high", "Organization": "OpenAI", "Release date": "2024-12-17", "Started at": "2026-08-28T07:09:23.000Z", "id": "NMJhDtxtyQJCHQ4EApr9LM", "mean_score": "0.14736842105263157", "stderr": "0.021034091168852485"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-13", "model": "o1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.947129909365559, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-2024-12-17_high", "source_row": {"Best score (across scorers)": "0.947129909365559", "Country": "United States of America", "Model version": "o1-2024-12-17_high", "Organization": "OpenAI", "Release date": "2024-12-17", "Started at": "2025-02-13T17:12:08.885Z", "id": "jVXtRkTBDnqH8p4sjVUbgN", "mean_score": "0.947129909365559", "stderr": "0.006152192005965586"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-27", "model": "o1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-2024-12-17_medium", "source_row": {"Best score (across scorers)": "0.7333333333333333", "Country": "United States of America", "Model version": "o1-2024-12-17_medium", "Organization": "OpenAI", "Release date": "2024-12-17", "Started at": "2025-02-27T14:16:13.444Z", "id": "KvcqycaJpCQMBigiEosyzC", "mean_score": "0.7333333333333333", "stderr": "0.06666666666666667"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-13", "model": "o1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7676767676767676, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-2024-12-17_high", "source_row": {"Best score (across scorers)": "0.7676767676767676", "Country": "United States of America", "Model version": "o1-2024-12-17_high", "Organization": "OpenAI", "Release date": "2024-12-17", "Started at": "2025-02-13T17:12:11.445Z", "id": "NAkCFiFiDSN3NXMj7HMvMS", "mean_score": "0.7676767676767676", "stderr": "0.030088629490217445"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "o1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.411, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-2024-12-17_high", "source_row": {"Best score (across scorers)": "0.411", "Country": "United States of America", "Model version": "o1-2024-12-17_high", "Organization": "OpenAI", "Release date": "2024-12-17", "Started at": "2026-08-31T22:01:34.000Z", "id": "LQfu4ngArcgMEyQZxuR7r4", "mean_score": "0.411", "stderr": "0.01556667341859915"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-29", "model": "deepseek-v3-1-terminus", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7942762, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v3.1-terminus", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "rows": [{"accuracy": "0.7969696000000001", "provider_name": "AtlasCloud", "total_questions": "990"}, {"accuracy": "0.7839820659898478", "provider_name": "DeepInfra", "total_questions": "591"}, {"accuracy": "0.8235687999999998", "provider_name": "Novita", "total_questions": "990"}, {"accuracy": "0.0025255000000000004", "provider_name": "SiliconFlow", "total_questions": "396"}, {"accuracy": "0.7942762", "provider_name": "StreamLake", "total_questions": "990"}], "rule": "median over providers (5 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-29", "model": "deepseek-v3-1-terminus", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.48266679999999995, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-v3.1-terminus", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-v3.1-terminus", "rows": [{"accuracy": "0.18666649999999999", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.5104164999999999", "provider_name": "DeepInfra", "total_questions": "96"}, {"accuracy": "0.7244622983870967", "provider_name": "Novita", "total_questions": "248"}, {"accuracy": "0.48266679999999995", "provider_name": "SiliconFlow", "total_questions": "250"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "gemini-2-5-flash-jun-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.738309, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-2.5-flash", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemini-2.5-flash", "rows": [{"accuracy": "0.738309", "provider_name": "Google", "total_questions": "1782"}, {"accuracy": "0.758698", "provider_name": "Google AI Studio", "total_questions": "594"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gemini-2-5-flash-jun-2025", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5624, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Gemini-2.5-Flash (FC)", "source_row": {"Model": "Gemini-2.5-Flash (FC)", "Organization": "Google", "Overall Acc": "56.24%", "Rank": "15"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-01", "model": "gemini-2-5-flash-jun-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.5511113333333334, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemini-2.5-flash", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemini-2.5-flash", "rows": [{"accuracy": "0.5881481111111111", "provider_name": "Google", "total_questions": "450"}, {"accuracy": "0.5511113333333334", "provider_name": "Google AI Studio", "total_questions": "150"}], "rule": "first-party provider (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-05-20", "model": "gemini-2-5-flash-may-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7083333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-flash-preview-05-20", "source_row": {"Best score (across scorers)": "0.7083333333333334", "Country": "United States of America", "Model version": "gemini-2.5-flash-preview-05-20", "Organization": "Google DeepMind", "Release date": "2025-05-20", "Started at": "2025-05-20T21:22:01.376Z", "id": "9e62UjoHQbcZD2xjgfRkdK", "mean_score": "0.7083333333333334", "stderr": "0.05715807427254223"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-05-25", "model": "gemini-2-5-flash-may-2025", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.551, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gemini-2.5-flash-preview-05-20 (24k think)", "source_row": {"command": "aider --model gemini/gemini-2.5-flash-preview-05-20", "date": "2025-05-25", "dirname": "2025-05-25-22-58-44--flash25-05-20-24k-think", "model": "gemini-2.5-flash-preview-05-20 (24k think)", "pass_rate_2": 55.1, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-22", "model": "gemini-2-5-flash-apr-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7305555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.5-flash-preview-04-17", "source_row": {"Best score (across scorers)": "0.7305555555555555", "Country": "United States of America", "Model version": "gemini-2.5-flash-preview-04-17", "Organization": "Google DeepMind", "Release date": "2025-04-17", "Started at": "2025-04-22T10:03:18.492Z", "id": "KpZYG5nvdJ6QYweLz6dn5a", "mean_score": "0.7305555555555555", "stderr": "0.05532781604070319"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-20", "model": "gemini-2-5-flash-apr-2025", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.47100000000000003, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gemini-2.5-flash-preview-04-17 (default)", "source_row": {"command": "aider --model gemini/gemini-2.5-flash-preview-04-17", "date": "2025-04-20", "dirname": "2025-04-20-19-54-31--flash25-diff-no-think", "model": "gemini-2.5-flash-preview-04-17 (default)", "pass_rate_2": 47.1, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-27", "model": "qwen3-5-9b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6166666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-9B_none", "source_row": {"Best score (across scorers)": "0.6166666666666667", "Country": "China", "Model version": "qwen3.5-9B_none", "Organization": "Alibaba", "Release date": "2026-02-24", "Started at": "2026-08-27T14:31:08.000Z", "id": "YBMTDnW5LmCN6tBp2QYHzi", "mean_score": "0.6166666666666667", "stderr": "0.06003996985174109"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-27", "model": "qwen3-5-9b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7897727272727273, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-9B", "source_row": {"Best score (across scorers)": "0.7897727272727273", "Country": "China", "Model version": "qwen3.5-9B", "Organization": "Alibaba", "Release date": "2026-02-24", "Started at": "2026-08-27T14:30:17.000Z", "id": "o6migB5WP27uLr9nUWpCiM", "mean_score": "0.7897727272727273", "stderr": "0.02365568030958479"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "qwen3-5-9b", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.7462993119730186, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-9b-20260310", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3.5-9b-20260310", "rows": [{"accuracy": "0.6848486000000001", "provider_name": "Darkbloom", "run_count": "5", "total_questions": "990"}, {"accuracy": "0.7462993119730186", "provider_name": "SiliconFlow", "run_count": "9", "total_questions": "1779"}, {"accuracy": "0.79250825", "provider_name": "Venice", "run_count": "12", "total_questions": "2376"}], "rule": "median over providers (3 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "qwen3-5-9b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.650067849593496, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3.5-9b-20260310", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3.5-9b-20260310", "rows": [{"accuracy": "0.7563351081871345", "provider_name": "DeepInfra", "total_questions": "342"}, {"accuracy": "0.650067849593496", "provider_name": "SiliconFlow", "total_questions": "492"}, {"accuracy": "0.7438479642058166", "provider_name": "Together", "total_questions": "447"}, {"accuracy": "0.6396396317567568", "provider_name": "Venice", "total_questions": "592"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-04-10", "model": "grok-3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9093655589123867, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-3-mini-beta_low", "source_row": {"Best score (across scorers)": "0.9093655589123867", "Country": "United States of America", "Model version": "grok-3-mini-beta_low", "Organization": "xAI", "Release date": "2025-04-09", "Started at": "2025-04-10T20:30:31.009Z", "id": "5QyCm49Lce2PpGVcZLomCz", "mean_score": "0.9093655589123867", "stderr": "0.007892888346623621"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-10", "model": "grok-3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7777777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-3-mini-beta_high", "source_row": {"Best score (across scorers)": "0.7777777777777778", "Country": "United States of America", "Model version": "grok-3-mini-beta_high", "Organization": "xAI", "Release date": "2025-04-09", "Started at": "2025-04-10T18:02:28.291Z", "id": "fPtize44vDPcDAzMFuAurw", "mean_score": "0.7777777777777778", "stderr": "0.06267511942419625"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-10", "model": "grok-3-mini", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.493, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Grok 3 Mini Beta (high)", "source_row": {"command": "aider --model xai/grok-3-mini-beta --reasoning-effort high", "date": "2025-04-10", "dirname": "2025-04-10-23-59-02--xai-grok3-mini-whole-high", "model": "Grok 3 Mini Beta (high)", "pass_rate_2": 49.3, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-10", "model": "grok-3-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7626262626262627, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-3-mini-beta_low", "source_row": {"Best score (across scorers)": "0.7626262626262627", "Country": "United States of America", "Model version": "grok-3-mini-beta_low", "Organization": "xAI", "Release date": "2025-04-09", "Started at": "2025-04-10T20:30:33.024Z", "id": "McxfQdC5aypnD3STwZUBjH", "mean_score": "0.7626262626262627", "stderr": "0.03031371053819892"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "deepseek-v3-1", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.7707982259154932, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-chat-v3.1", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3.1", "rows": [{"accuracy": "0.798541", "provider_name": "AtlasCloud", "run_count": "6", "total_questions": "1188"}, {"accuracy": "0.7828282222222223", "provider_name": "CoreWeave", "run_count": "9", "total_questions": "1782"}, {"accuracy": "0.7707982259154932", "provider_name": "DeepInfra", "run_count": "9", "total_questions": "1775"}, {"accuracy": "0.75084367721519", "provider_name": "Google", "run_count": "4", "total_questions": "790"}, {"accuracy": "0.815488", "provider_name": "Novita", "run_count": "5", "total_questions": "990"}, {"accuracy": "0.001684", "provider_name": "SiliconFlow", "run_count": "2", "total_questions": "396"}], "rule": "median over providers (6 of 6 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-15", "model": "deepseek-v3-1", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.47222181249999995, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-chat-v3.1", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3.1", "rows": [{"accuracy": "0.39210027167630057", "provider_name": "CoreWeave", "total_questions": "346"}, {"accuracy": "0.42196971363636365", "provider_name": "DeepInfra", "total_questions": "440"}, {"accuracy": "0.47222181249999995", "provider_name": "Google", "total_questions": "96"}, {"accuracy": "0.7207207364864865", "provider_name": "Novita", "total_questions": "296"}, {"accuracy": "0.5282739330357142", "provider_name": "SiliconFlow", "total_questions": "448"}], "rule": "median over providers (5 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "qwen3-30b-a3b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7027777777777777, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-30b-a3b-thinking-2507", "source_row": {"Best score (across scorers)": "0.7027777777777777", "Country": "China", "Model version": "qwen3-30b-a3b-thinking-2507", "Organization": "Alibaba", "Release date": "2025-07-30", "Started at": "2026-08-30T22:44:45.000Z", "Training compute (FLOP)": "6.48e+23", "id": "6rZFxNR3GjfhWey6DzEiYz", "mean_score": "0.7027777777777777", "stderr": "0.060418480757624834"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "qwen3-30b-a3b-thinking-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7007575757575758, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-30b-a3b-thinking-2507", "source_row": {"Best score (across scorers)": "0.7007575757575758", "Country": "China", "Model version": "qwen3-30b-a3b-thinking-2507", "Organization": "Alibaba", "Release date": "2025-07-30", "Started at": "2026-08-30T22:44:24.000Z", "Training compute (FLOP)": "6.48e+23", "id": "2gzNF2d53h2oCKDnojLtSP", "mean_score": "0.7007575757575758", "stderr": "0.028327897613545493"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "seed-oss-36b-instruct", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.675, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "seed-oss-36b-instruct", "source_row": {"Best score (across scorers)": "0.675", "Model version": "seed-oss-36b-instruct", "Release date": "2025-08-20", "Started at": "2026-08-28T18:40:22.000Z", "id": "eJvV6RezECZwh3b4ekg4pm", "mean_score": "0.675", "stderr": "0.06324056203789373"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "seed-oss-36b-instruct", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7152777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "seed-oss-36b-instruct", "source_row": {"Best score (across scorers)": "0.7152777777777778", "Model version": "seed-oss-36b-instruct", "Release date": "2025-08-20", "Started at": "2026-08-30T22:46:22.000Z", "id": "BVat526nF6NUGoVaLCp7DT", "mean_score": "0.7152777777777778", "stderr": "0.02741123270979468"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-31", "model": "deepseek-r1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.9305135951661632, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1", "source_row": {"Best score (across scorers)": "0.9305135951661632", "Country": "China", "Model version": "DeepSeek-R1", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-01-31T23:48:19.611Z", "Training compute (FLOP)": "3.5e+24", "id": "XVqLs2khGFSNwEbHJTEgRU", "mean_score": "0.9305135951661632", "stderr": "0.006990872160031429"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-26", "model": "deepseek-r1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1", "source_row": {"Best score (across scorers)": "0.5333333333333333", "Country": "China", "Model version": "DeepSeek-R1", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-02-26T16:56:45.594Z", "Training compute (FLOP)": "3.5e+24", "id": "bBY5VZzpDkmTBdSSvu5Ups", "mean_score": "0.5333333333333333", "stderr": "0.07521014330903548"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-01-20", "model": "deepseek-r1", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.569, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "DeepSeek R1", "source_row": {"command": "aider --model deepseek/deepseek-reasoner", "date": "2025-01-20", "dirname": "2025-01-20-19-11-38--ds-turns-upd-cur-msgs-fix-with-summarizer", "model": "DeepSeek R1", "pass_rate_2": 56.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-05-26", "model": "deepseek-r1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7171717171717171, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1", "source_row": {"Best score (across scorers)": "0.7171717171717171", "Country": "China", "Model version": "DeepSeek-R1", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-05-26T18:38:03.702Z", "Training compute (FLOP)": "3.5e+24", "id": "fFGyQR2mfDufvrSV2rzWyV", "mean_score": "0.6922348484848484", "stderr": "0.03070485693321271"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-03", "model": "qwen3-235b-a22b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7388167142857143, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-235b-a22b-07-25", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "rows": [{"accuracy": "0.7388167142857143", "provider_name": "Alibaba", "total_questions": "1386"}, {"accuracy": "0.7107745999999999", "provider_name": "AtlasCloud", "total_questions": "990"}, {"accuracy": "0.76638602247191", "provider_name": "Crusoe", "total_questions": "356"}, {"accuracy": "0.7209128888888889", "provider_name": "DeepInfra", "total_questions": "1782"}, {"accuracy": "0.7163299999999999", "provider_name": "Friendli", "total_questions": "198"}, {"accuracy": "0.7028622499999999", "provider_name": "GMICloud", "total_questions": "792"}, {"accuracy": "0.7340773735119047", "provider_name": "Google", "total_questions": "3360"}, {"accuracy": "0.743266", "provider_name": "Nebius", "total_questions": "1584"}, {"accuracy": "0.7281143333333333", "provider_name": "Novita", "total_questions": "1188"}, {"accuracy": "0.7361669504685407", "provider_name": "Parasail", "total_questions": "1494"}, {"accuracy": "0.7525250000000001", "provider_name": "PrimeIntellect", "total_questions": "198"}, {"accuracy": "0.744318125", "provider_name": "StreamLake", "total_questions": "1584"}, {"accuracy": "0.7297980000000001", "provider_name": "Venice", "total_questions": "1386"}], "rule": "first-party provider (1 of 13 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-235b-a22b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.5215, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", "source_row": {"Model": "Qwen3-235B-A22B-Instruct-2507 (Prompt)", "Organization": "Qwen", "Overall Acc": "52.15%", "Rank": "23"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-03", "model": "qwen3-235b-a22b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.3961905714285714, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-235b-a22b-07-25", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-235b-a22b-07-25", "rows": [{"accuracy": "0.3961905714285714", "provider_name": "Alibaba", "total_questions": "350"}, {"accuracy": "0.4222225", "provider_name": "AtlasCloud", "total_questions": "300"}, {"accuracy": "0.47916700000000007", "provider_name": "Crusoe", "total_questions": "96"}, {"accuracy": "0.41901883467741935", "provider_name": "DeepInfra", "total_questions": "496"}, {"accuracy": "0.36833325", "provider_name": "GMICloud", "total_questions": "200"}, {"accuracy": "0.45183054913294796", "provider_name": "Google", "total_questions": "346"}, {"accuracy": "0.45666666666666667", "provider_name": "Nebius", "total_questions": "450"}, {"accuracy": "0.4238095714285714", "provider_name": "Novita", "total_questions": "350"}, {"accuracy": "0.4563758489932886", "provider_name": "Parasail", "total_questions": "298"}, {"accuracy": "0.406667", "provider_name": "PrimeIntellect", "total_questions": "50"}, {"accuracy": "0.453333375", "provider_name": "StreamLake", "total_questions": "400"}, {"accuracy": "0.44708325000000004", "provider_name": "Venice", "total_questions": "400"}], "rule": "first-party provider (1 of 12 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-04-10", "model": "grok-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8874622356495468, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-3-beta", "source_row": {"Best score (across scorers)": "0.8874622356495468", "Country": "United States of America", "Model version": "grok-3-beta", "Organization": "xAI", "Release date": "2025-04-09", "Started at": "2025-04-10T17:55:22.706Z", "Training compute (FLOP)": "3.5e+26", "id": "i9eqMbaVTDDuCxWxzGHxyU", "mean_score": "0.8874622356495468", "stderr": "0.008688487093324447"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-10", "model": "grok-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-3-beta", "source_row": {"Best score (across scorers)": "0.5555555555555556", "Country": "United States of America", "Model version": "grok-3-beta", "Organization": "xAI", "Release date": "2025-04-09", "Started at": "2025-04-10T17:55:19.398Z", "Training compute (FLOP)": "3.5e+26", "id": "DKTzh9kSoipZGkDsC4NGf3", "mean_score": "0.5555555555555556", "stderr": "0.07491109582924914"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-10", "model": "grok-3", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.5329999999999999, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Grok 3 Beta", "source_row": {"command": "aider --model openrouter/x-ai/grok-3-beta", "date": "2025-04-10", "dirname": "2025-04-10-04-21-31--grok3-diff-exuser", "model": "Grok 3 Beta", "pass_rate_2": 53.3, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-05-26", "model": "grok-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7575757575757576, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-3-beta", "source_row": {"Best score (across scorers)": "0.7575757575757576", "Country": "United States of America", "Model version": "grok-3-beta", "Organization": "xAI", "Release date": "2025-04-09", "Started at": "2025-05-26T17:42:41.696Z", "Training compute (FLOP)": "3.5e+26", "id": "FQ8Ykq4JWRSXnyF9rbmLc7", "mean_score": "0.6758207070707071", "stderr": "0.026863078746340353"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "qwen3-5-4b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5583333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3.5-4b_none", "source_row": {"Best score (across scorers)": "0.5583333333333333", "Country": "China", "Model version": "qwen3.5-4b_none", "Organization": "Alibaba", "Release date": "2026-02-24", "Started at": "2026-08-28T18:33:35.000Z", "id": "SAZy8rk2HgyPjA8XZJFUmR", "mean_score": "0.5583333333333333", "stderr": "0.0616226296489243"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-09-17", "model": "mistral-small-4", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.7528056666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "mistralai/mistral-small-2603", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "mistralai/mistral-small-2603", "rows": [{"accuracy": "0.7528056666666667", "provider_name": "Mistral", "total_questions": "1188"}, {"accuracy": "0.759259", "provider_name": "Venice", "total_questions": "396"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-09-17", "model": "mistral-small-4", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.41142871428571426, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "mistralai/mistral-small-2603", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "mistralai/mistral-small-2603", "rows": [{"accuracy": "0.41142871428571426", "provider_name": "Mistral", "total_questions": "350"}], "rule": "first-party provider (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-27", "model": "gpt-oss-20b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6527777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-oss-20b_medium", "source_row": {"Best score (across scorers)": "0.6527777777777778", "Country": "United States of America", "Model version": "gpt-oss-20b_medium", "Organization": "OpenAI", "Release date": "2025-08-05", "Started at": "2026-08-27T14:12:40.000Z", "Training compute (FLOP)": "5.49e+23", "id": "DB8ja6pEZWGocQbdciyisL", "mean_score": "0.6527777777777778", "stderr": "0.0593111883162336"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-27", "model": "gpt-oss-20b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6079545454545454, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-oss-20b_medium", "source_row": {"Best score (across scorers)": "0.6079545454545454", "Country": "United States of America", "Model version": "gpt-oss-20b_medium", "Organization": "OpenAI", "Release date": "2025-08-05", "Started at": "2026-08-27T14:06:50.000Z", "Training compute (FLOP)": "5.49e+23", "id": "GEwyLqZT93Hp55fdtHrJhm", "mean_score": "0.6079545454545454", "stderr": "0.024864059770258008"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "gpt-oss-20b", "retrieved_at": "2026-10-18T13:52:14Z", "score": 0.6311448, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-oss-20b", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-oss-20b", "rows": [{"accuracy": "0.6576179999999999", "provider_name": "Amazon Bedrock", "run_count": "16", "total_questions": "3168"}, {"accuracy": "0.6554433333333334", "provider_name": "CoreWeave", "run_count": "9", "total_questions": "1782"}, {"accuracy": "0.6311448", "provider_name": "Darkbloom", "run_count": "5", "total_questions": "990"}, {"accuracy": "0.695286", "provider_name": "DekaLLM", "run_count": "1", "total_questions": "198"}, {"accuracy": "0.5774414999999999", "provider_name": "Google", "run_count": "2", "total_questions": "396"}, {"accuracy": "0.6338480836550836", "provider_name": "Groq", "run_count": "8", "total_questions": "1554"}, {"accuracy": "0.5906884318766067", "provider_name": "Novita", "run_count": "6", "total_questions": "1167"}, {"accuracy": "0.6582491666666667", "provider_name": "Parasail", "run_count": "6", "total_questions": "1188"}, {"accuracy": "0.599327", "provider_name": "PrimeIntellect", "run_count": "1", "total_questions": "198"}, {"accuracy": "0.6272744175576814", "provider_name": "SiliconFlow", "run_count": "9", "total_questions": "1777"}], "rule": "median over providers (10 of 10 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-17", "model": "gpt-oss-20b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.4672696511627907, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-oss-20b", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-oss-20b", "rows": [{"accuracy": "0.408060476070529", "provider_name": "Amazon Bedrock", "total_questions": "794"}, {"accuracy": "0.28901757803468203", "provider_name": "CoreWeave", "total_questions": "346"}, {"accuracy": "0.5438595101214574", "provider_name": "Darkbloom", "total_questions": "247"}, {"accuracy": "0.5051021836734694", "provider_name": "DeepInfra", "total_questions": "196"}, {"accuracy": "0.4672696511627907", "provider_name": "Groq", "total_questions": "387"}, {"accuracy": "0.4707204932432433", "provider_name": "Parasail", "total_questions": "296"}, {"accuracy": "0.333333", "provider_name": "PrimeIntellect", "total_questions": "50"}], "rule": "median over providers (7 of 7 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "glm-4-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5833333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7-flash", "source_row": {"Best score (across scorers)": "0.5833333333333334", "Model version": "glm-4.7-flash", "Started at": "2026-08-30T22:37:43.000Z", "id": "9ekki5tJhyjmA7ZMLoUxzo", "mean_score": "0.5833333333333334", "stderr": "0.06564078093085213"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "glm-4-7-flash", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6054292929292929, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7-flash", "source_row": {"Best score (across scorers)": "0.6054292929292929", "Model version": "glm-4.7-flash", "Started at": "2026-08-30T22:37:35.000Z", "id": "GL7htq7XRnbzqpAVK5f862", "mean_score": "0.6054292929292929", "stderr": "0.02934798760577577"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-30", "model": "glm-4-7-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.49197393750000007, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-4.7-flash-20260119", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "rows": [{"accuracy": "0.5841753333333334", "provider_name": "Cloudflare", "total_questions": "1188"}, {"accuracy": "0.49197393750000007", "provider_name": "DeepInfra", "total_questions": "976"}, {"accuracy": "0.5816397652439025", "provider_name": "Novita", "total_questions": "984"}, {"accuracy": "0.48962583163265305", "provider_name": "Venice", "total_questions": "980"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-30", "model": "glm-4-7-flash", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.6833333333333333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "z-ai/glm-4.7-flash-20260119", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "z-ai/glm-4.7-flash-20260119", "rows": [{"accuracy": "0.6733331428571429", "provider_name": "Cloudflare", "total_questions": "350"}, {"accuracy": "0.6872909230769231", "provider_name": "DeepInfra", "total_questions": "299"}, {"accuracy": "0.6833333333333333", "provider_name": "Novita", "total_questions": "300"}, {"accuracy": "0.6923076113360324", "provider_name": "Venice", "total_questions": "247"}], "rule": "median over providers (4 of 4 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "qwen3-30b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6277777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-30B-A3B", "source_row": {"Best score (across scorers)": "0.6277777777777778", "Country": "China", "Model version": "Qwen3-30B-A3B", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-28T17:03:13.000Z", "Training compute (FLOP)": "6.48e+23", "id": "XfjRkSyP4ub4gDsWcgS7KY", "mean_score": "0.6277777777777778", "stderr": "0.062118645754659016"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "qwen3-30b-a3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6167929292929293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Qwen3-30B-A3B", "source_row": {"Best score (across scorers)": "0.6167929292929293", "Country": "China", "Model version": "Qwen3-30B-A3B", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-30T22:43:15.000Z", "Training compute (FLOP)": "6.48e+23", "id": "BnX67AoZzgVbHgaM8F8Brz", "mean_score": "0.6167929292929293", "stderr": "0.029727711361608088"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "qwen3-30b-a3b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.604377, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-30b-a3b-04-28", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "rows": [{"accuracy": "0.604377", "provider_name": "Alibaba", "total_questions": "198"}, {"accuracy": "0.608586", "provider_name": "DeepInfra", "total_questions": "396"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "qwen3-30b-a3b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.423333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-30b-a3b-04-28", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-04-28", "rows": [{"accuracy": "0.423333", "provider_name": "Alibaba", "total_questions": "100"}, {"accuracy": "0.46166650000000004", "provider_name": "DeepInfra", "total_questions": "100"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "cohere-command-a", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.46490000000000004, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Command A (FC)", "source_row": {"Model": "Command A (FC)", "Organization": "Cohere", "Overall Acc": "46.49%", "Rank": "35"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "deepseek-r1-distill-qwen-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Qwen-32B", "source_row": {"Best score (across scorers)": "0.5555555555555556", "Country": "China", "Model version": "DeepSeek-R1-Distill-Qwen-32B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2026-08-28T08:15:43.000Z", "id": "KbUNQf8nnwPa8ixg6AnEwB", "mean_score": "0.5555555555555556", "stderr": "0.061999963795652153"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "deepseek-r1-distill-qwen-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6414141414141414, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Qwen-32B", "source_row": {"Best score (across scorers)": "0.6414141414141414", "Country": "China", "Model version": "DeepSeek-R1-Distill-Qwen-32B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2026-08-28T08:15:19.000Z", "id": "dwyb2yzFewGT6w6oFdVMAc", "mean_score": "0.6414141414141414", "stderr": "0.027534177016461762"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "qwen3-14b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6638888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-14b", "source_row": {"Best score (across scorers)": "0.6638888888888889", "Country": "China", "Model version": "qwen3-14b", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-28T16:54:14.000Z", "Training compute (FLOP)": "3.1968e+24", "id": "gRRQHECkYhz5UaYsRKDh2F", "mean_score": "0.6638888888888889", "stderr": "0.06190655406811223"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "qwen3-14b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6376262626262627, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-14b", "source_row": {"Best score (across scorers)": "0.6376262626262627", "Country": "China", "Model version": "qwen3-14b", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-28T16:53:49.000Z", "Training compute (FLOP)": "3.1968e+24", "id": "XGK9DjJHHVWr2LyT298ooL", "mean_score": "0.6376262626262627", "stderr": "0.02931953778086544"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "qwen3-14b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.5331086666666667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-14b-04-28", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-14b-04-28", "rows": [{"accuracy": "0.5331086666666667", "provider_name": "Alibaba", "total_questions": "594"}, {"accuracy": "0.6128760749682338", "provider_name": "DeepInfra", "total_questions": "787"}, {"accuracy": "0.5973626666666666", "provider_name": "NextBit", "total_questions": "1188"}], "rule": "first-party provider (1 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-14b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.4103, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-14B (FC)", "source_row": {"Model": "Qwen3-14B (FC)", "Organization": "Qwen", "Overall Acc": "41.03%", "Rank": "43"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "qwen3-14b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.413333, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-14b-04-28", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-14b-04-28", "rows": [{"accuracy": "0.413333", "provider_name": "Alibaba", "total_questions": "50"}, {"accuracy": "0.43083325", "provider_name": "DeepInfra", "total_questions": "200"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-03-10", "model": "deepseek-r1-distill-llama-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8989803625377644, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Llama-70B", "source_row": {"Best score (across scorers)": "0.8989803625377644", "Country": "China", "Model version": "DeepSeek-R1-Distill-Llama-70B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-03-10T22:53:10.180Z", "id": "oPDoTJFMhMNvU4sybYu2jQ", "mean_score": "0.8989803625377644", "stderr": "0.006818015200174493"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-07", "model": "deepseek-r1-distill-llama-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5138888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Llama-70B", "source_row": {"Best score (across scorers)": "0.5138888888888888", "Country": "China", "Model version": "DeepSeek-R1-Distill-Llama-70B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-03-07T23:03:59.002Z", "id": "89zcekqEYnE8zhwG4pR5Nu", "mean_score": "0.5138888888888888", "stderr": "0.060540263104731575"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-03-10", "model": "deepseek-r1-distill-llama-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5574494949494949, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Llama-70B", "source_row": {"Best score (across scorers)": "0.5574494949494949", "Country": "China", "Model version": "DeepSeek-R1-Distill-Llama-70B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-03-10T20:08:15.034Z", "id": "f6pnAATjxmrFJS353EjQKi", "mean_score": "0.5574494949494949", "stderr": "0.02973465219179067"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "qwq-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5916666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "QwQ-32B", "source_row": {"Best score (across scorers)": "0.5916666666666667", "Country": "China", "Model version": "QwQ-32B", "Organization": "Alibaba", "Release date": "2025-03-05", "Started at": "2026-08-28T07:20:55.000Z", "Training compute (FLOP)": "3.51e+24", "id": "2xMn6dbY9mw4pmkh8Tz9Gn", "mean_score": "0.5916666666666667", "stderr": "0.06348963965431165"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-03-06", "model": "qwq-32b", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.209, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "QwQ-32B", "source_row": {"command": "aider --model fireworks_ai/accounts/fireworks/models/qwq-32b", "date": "2025-03-06", "dirname": "2025-03-06-17-40-24--qwq32b-diff-temp-topp-ex-sys-remind-user-for-real", "model": "QwQ-32B", "pass_rate_2": 20.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "qwq-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6534090909090909, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "QwQ-32B", "source_row": {"Best score (across scorers)": "0.6534090909090909", "Country": "China", "Model version": "QwQ-32B", "Organization": "Alibaba", "Release date": "2025-03-05", "Started at": "2026-08-28T07:20:47.000Z", "Training compute (FLOP)": "3.51e+24", "id": "PUXMxQiRY6XaG6NjwsEDLy", "mean_score": "0.6534090909090909", "stderr": "0.0281399081227253"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-13", "model": "o1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8918051359516617, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-mini-2024-09-12_high", "source_row": {"Best score (across scorers)": "0.8918051359516617", "Country": "United States of America", "Model version": "o1-mini-2024-09-12_high", "Organization": "OpenAI", "Release date": "2024-09-12", "Started at": "2025-02-13T17:07:52.489Z", "id": "4DWnpJowAfeQDHc2xDNy2C", "mean_score": "0.8918051359516617", "stderr": "0.006802820166065648"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-06", "model": "o1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46944444444444444, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-mini-2024-09-12_high", "source_row": {"Best score (across scorers)": "0.46944444444444444", "Country": "United States of America", "Model version": "o1-mini-2024-09-12_high", "Organization": "OpenAI", "Release date": "2024-09-12", "Started at": "2025-03-06T21:47:29.432Z", "id": "9zhPZLvNeUPQ7JBSpTwKPp", "mean_score": "0.46944444444444444", "stderr": "0.06456873681038247"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-22", "model": "o1-mini", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.32899999999999996, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "o1-mini-2024-09-12", "source_row": {"command": "aider --model o1-mini", "date": "2024-12-22", "dirname": "2024-12-22-21-26-35--polyglot-o1mini-whole", "model": "o1-mini-2024-09-12", "pass_rate_2": 32.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-13", "model": "o1-mini", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6237373737373737, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-mini-2024-09-12_high", "source_row": {"Best score (across scorers)": "0.6237373737373737", "Country": "United States of America", "Model version": "o1-mini-2024-09-12_high", "Organization": "OpenAI", "Release date": "2024-09-12", "Started at": "2025-02-13T17:07:50.981Z", "id": "X2gKewiGPZx5DLFEddUUqW", "mean_score": "0.6237373737373737", "stderr": "0.027427723511230753"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "qwen3-30b-a3b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6222222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-30b-a3b-instruct-2507", "source_row": {"Best score (across scorers)": "0.6222222222222222", "Country": "China", "Model version": "qwen3-30b-a3b-instruct-2507", "Organization": "Alibaba", "Release date": "2025-07-29", "Started at": "2026-08-30T22:44:11.000Z", "Training compute (FLOP)": "6.48e+23", "id": "Eo5cAQJK28CHpKDfCSDYG4", "mean_score": "0.6222222222222222", "stderr": "0.06509619058885623"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "qwen3-30b-a3b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5561868686868687, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-30b-a3b-instruct-2507", "source_row": {"Best score (across scorers)": "0.5561868686868687", "Country": "China", "Model version": "qwen3-30b-a3b-instruct-2507", "Organization": "Alibaba", "Release date": "2025-07-29", "Started at": "2026-08-30T22:44:04.000Z", "Training compute (FLOP)": "6.48e+23", "id": "X3HfncZtnLKUPoUuRc9YsR", "mean_score": "0.5561868686868687", "stderr": "0.028844434239616544"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-03", "model": "qwen3-30b-a3b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6419754999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-30b-a3b-instruct-2507", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "rows": [{"accuracy": "0.6419754999999999", "provider_name": "Alibaba", "total_questions": "1188"}, {"accuracy": "0.641414", "provider_name": "Ambient", "total_questions": "198"}, {"accuracy": "0.6308680607902736", "provider_name": "CoreWeave", "total_questions": "987"}, {"accuracy": "0.6502977410714286", "provider_name": "DekaLLM", "total_questions": "224"}, {"accuracy": "0.6336718477157359", "provider_name": "Nebius", "total_questions": "1182"}, {"accuracy": "0.621212", "provider_name": "PrimeIntellect", "total_questions": "198"}, {"accuracy": "0.6368495215189873", "provider_name": "SiliconFlow", "total_questions": "1185"}, {"accuracy": "0.6334780000000001", "provider_name": "StreamLake", "total_questions": "1386"}], "rule": "first-party provider (1 of 8 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-30b-a3b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.4139, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-30B-A3B-Instruct-2507 (FC)", "source_row": {"Model": "Qwen3-30B-A3B-Instruct-2507 (FC)", "Organization": "Qwen", "Overall Acc": "41.39%", "Rank": "41"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-03", "model": "qwen3-30b-a3b-instruct-jul-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.3695237142857143, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-30b-a3b-instruct-2507", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-30b-a3b-instruct-2507", "rows": [{"accuracy": "0.3695237142857143", "provider_name": "Alibaba", "total_questions": "350"}, {"accuracy": "0.386667", "provider_name": "Ambient", "total_questions": "50"}, {"accuracy": "0.39", "provider_name": "CoreWeave", "total_questions": "300"}, {"accuracy": "0.372222", "provider_name": "Nebius", "total_questions": "300"}, {"accuracy": "0.36", "provider_name": "PrimeIntellect", "total_questions": "50"}, {"accuracy": "0.39809542857142854", "provider_name": "SiliconFlow", "total_questions": "350"}, {"accuracy": "0.40897435897435896", "provider_name": "StreamLake", "total_questions": "390"}], "rule": "first-party provider (1 of 7 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-27", "model": "qwen3-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5611111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-8b", "source_row": {"Best score (across scorers)": "0.5611111111111111", "Country": "China", "Model version": "qwen3-8b", "Organization": "Alibaba", "Release date": "2025-04-28", "Started at": "2026-08-27T14:21:53.000Z", "Training compute (FLOP)": "1.7712e+24", "id": "gG2B6fLUDBy5jRraxL2pz8", "mean_score": "0.5611111111111111", "stderr": "0.0645035213369854"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-27", "model": "qwen3-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5675505050505051, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-8b", "source_row": {"Best score (across scorers)": "0.5675505050505051", "Country": "China", "Model version": "qwen3-8b", "Organization": "Alibaba", "Release date": "2025-04-28", "Started at": "2026-08-27T14:21:43.000Z", "Training compute (FLOP)": "1.7712e+24", "id": "jRiWmBxtBAyiXiRBRMY8wX", "mean_score": "0.5675505050505051", "stderr": "0.029327713464985333"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-8b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.4257, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-8B (FC)", "source_row": {"Model": "Qwen3-8B (FC)", "Organization": "Qwen", "Overall Acc": "42.57%", "Rank": "39"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "deepseek-r1-0528-qwen3-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4388888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-r1-0528-qwen3-8b", "source_row": {"Best score (across scorers)": "0.4388888888888889", "Model version": "deepseek-r1-0528-qwen3-8b", "Release date": "2025-05-28", "Started at": "2026-08-28T07:49:02.000Z", "id": "LWbCXNGMXtjQYxyM2uoFMV", "mean_score": "0.4388888888888889", "stderr": "0.0677262181766212"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "deepseek-r1-0528-qwen3-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0928030303030303, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "deepseek-r1-0528-qwen3-8b", "source_row": {"Best score (across scorers)": "0.0928030303030303", "Model version": "deepseek-r1-0528-qwen3-8b", "Release date": "2025-05-28", "Started at": "2026-08-28T07:48:47.000Z", "id": "dszRh7ELyCEukWnwYnUxMp", "mean_score": "0.0928030303030303", "stderr": "0.010663485573693416"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-11", "model": "qwq-plus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6540404040404041, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwq-plus", "source_row": {"Best score (across scorers)": "0.6540404040404041", "Country": "China", "Model version": "qwq-plus", "Organization": "Alibaba", "Release date": "2025-04-08", "Started at": "2025-04-11T15:34:21.181Z", "id": "PtxHF8BH8WuNdNfDnutGBb", "mean_score": "0.6540404040404041", "stderr": "0.028469238770829935"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-03-11", "model": "deepseek-r1-distill-qwen-14b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8712235649546828, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Qwen-14B", "source_row": {"Best score (across scorers)": "0.8712235649546828", "Country": "China", "Model version": "DeepSeek-R1-Distill-Qwen-14B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-03-11T00:50:44.877Z", "id": "R8XGt5ScHpmTbD5DnXR59V", "mean_score": "0.8712235649546828", "stderr": "0.007486217204155208"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "deepseek-r1-distill-qwen-14b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5055555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Qwen-14B", "source_row": {"Best score (across scorers)": "0.5055555555555555", "Country": "China", "Model version": "DeepSeek-R1-Distill-Qwen-14B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2026-08-28T08:02:29.000Z", "id": "EceR3NeHzSxacdmAmfRxgT", "mean_score": "0.5055555555555555", "stderr": "0.05984981671790839"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-03-10", "model": "deepseek-r1-distill-qwen-14b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.44696969696969696, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Qwen-14B", "source_row": {"Best score (across scorers)": "0.44696969696969696", "Country": "China", "Model version": "DeepSeek-R1-Distill-Qwen-14B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2025-03-10T23:03:48.213Z", "id": "8UqaLobhmNJLuPGTTpeaCw", "mean_score": "0.44696969696969696", "stderr": "0.02863094678190378"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-06", "model": "gemini-2-0-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8345921450151057, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.0-pro-exp-02-05", "source_row": {"Best score (across scorers)": "0.8345921450151057", "Country": "United States of America", "Model version": "gemini-2.0-pro-exp-02-05", "Organization": "Google DeepMind", "Release date": "2025-02-05", "Started at": "2025-02-06T17:07:01.731Z", "id": "fvDt6HaHV5d3gSEuavixsv", "mean_score": "0.8345921450151057", "stderr": "0.01021492751504924"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-02-25", "model": "gemini-2-0-pro", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.35600000000000004, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Gemini 2.0 Pro exp-02-05", "source_row": {"command": "aider --model gemini/gemini-2.0-pro-exp-02-05", "date": "2025-02-25", "dirname": "2025-02-25-20-23-07--gemini-pro", "model": "Gemini 2.0 Pro exp-02-05", "pass_rate_2": 35.6, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-07", "model": "gemini-2-0-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6565656565656566, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.0-pro-exp-02-05", "source_row": {"Best score (across scorers)": "0.6565656565656566", "Country": "United States of America", "Model version": "gemini-2.0-pro-exp-02-05", "Organization": "Google DeepMind", "Release date": "2025-02-05", "Started at": "2025-02-07T07:28:03.988Z", "id": "g5YE7G6DXb45EH4HxEdiht", "mean_score": "0.6565656565656566", "stderr": "0.03383201223244441"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-07", "model": "gemini-2-0-flash-thinking-jan-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5777777777777777, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.0-flash-thinking-exp-01-21", "source_row": {"Best score (across scorers)": "0.5777777777777777", "Country": "United States of America", "Model version": "gemini-2.0-flash-thinking-exp-01-21", "Organization": "Google DeepMind,Google", "Release date": "2025-01-21", "Started at": "2025-03-07T22:18:52.419Z", "id": "cXz2gsCmGYSvQ6Ey4LDiN7", "mean_score": "0.5777777777777777", "stderr": "0.07446027270295805"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-01-21", "model": "gemini-2-0-flash-thinking-jan-2025", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.182, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gemini-2.0-flash-thinking-exp-01-21", "source_row": {"command": "aider --model gemini/gemini-2.0-flash-thinking-exp-01-21", "date": "2025-01-21", "dirname": "2025-01-21-22-51-49--gemini-2.0-flash-thinking-exp-01-21-polyglot-diff", "model": "gemini-2.0-flash-thinking-exp-01-21", "pass_rate_2": 18.2, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-06", "model": "gemini-2-0-flash-thinking-jan-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5707070707070707, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.0-flash-thinking-exp-01-21", "source_row": {"Best score (across scorers)": "0.5707070707070707", "Country": "United States of America", "Model version": "gemini-2.0-flash-thinking-exp-01-21", "Organization": "Google DeepMind,Google", "Release date": "2025-01-21", "Started at": "2025-02-06T22:40:04.398Z", "id": "ag9w2w4xa47ghZaWt5oZey", "mean_score": "0.5707070707070707", "stderr": "0.03526552724601199"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-28", "model": "gpt-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7862537764350453, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.5-preview-2025-02-27", "source_row": {"Best score (across scorers)": "0.7862537764350453", "Country": "United States of America", "Model version": "gpt-4.5-preview-2025-02-27", "Organization": "OpenAI", "Release date": "2025-02-27", "Started at": "2025-02-28T16:37:00.565Z", "Training compute (FLOP)": "3.8e+26", "id": "japqZFi3FS7um32EHG6wwW", "mean_score": "0.7862537764350453", "stderr": "0.011270694428136297"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-28", "model": "gpt-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37777777777777777, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.5-preview-2025-02-27", "source_row": {"Best score (across scorers)": "0.37777777777777777", "Country": "United States of America", "Model version": "gpt-4.5-preview-2025-02-27", "Organization": "OpenAI", "Release date": "2025-02-27", "Started at": "2025-02-28T09:17:11.571Z", "Training compute (FLOP)": "3.8e+26", "id": "9DCJMt7YsTMDenTdX4Db6V", "mean_score": "0.37777777777777777", "stderr": "0.07309112127323451"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-28", "model": "gpt-4-5", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6868686868686869, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4.5-preview-2025-02-27", "source_row": {"Best score (across scorers)": "0.6868686868686869", "Country": "United States of America", "Model version": "gpt-4.5-preview-2025-02-27", "Organization": "OpenAI", "Release date": "2025-02-27", "Started at": "2025-02-28T09:17:12.859Z", "Training compute (FLOP)": "3.8e+26", "id": "dodibSv7pk8GtqeMy4YLQg", "mean_score": "0.6868686868686869", "stderr": "0.033042050878136546"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-06", "model": "gemini-2-0-flash-feb-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8216578549848943, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.0-flash-001", "source_row": {"Best score (across scorers)": "0.8216578549848943", "Country": "United States of America", "Model version": "gemini-2.0-flash-001", "Organization": "Google DeepMind,Google", "Release date": "2025-02-05", "Started at": "2025-02-06T15:51:21.031Z", "id": "SqwGZz2UfTmbvPiw2rq3LH", "mean_score": "0.8216578549848943", "stderr": "0.008734081505176936"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gemini-2-0-flash-feb-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.0-flash-001", "source_row": {"Best score (across scorers)": "0.3111111111111111", "Country": "United States of America", "Model version": "gemini-2.0-flash-001", "Organization": "Google DeepMind,Google", "Release date": "2025-02-05", "Started at": "2025-02-25T12:25:50.002Z", "id": "ZnGSRrKbiEnuju4kk4sgVb", "mean_score": "0.3111111111111111", "stderr": "0.06311218144454454"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-06", "model": "gemini-2-0-flash-feb-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6414141414141414, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-2.0-flash-001", "source_row": {"Best score (across scorers)": "0.6414141414141414", "Country": "United States of America", "Model version": "gemini-2.0-flash-001", "Organization": "Google DeepMind,Google", "Release date": "2025-02-05", "Started at": "2025-02-06T15:51:21.852Z", "id": "fbszwTApm3f28z5VA6oXBK", "mean_score": "0.6414141414141414", "stderr": "0.027395618179027574"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-05-07", "model": "mistral-medium-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8162764350453172, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-medium-2505", "source_row": {"Best score (across scorers)": "0.8162764350453172", "Country": "France", "Model version": "mistral-medium-2505", "Organization": "Mistral AI", "Release date": "2025-05-07", "Started at": "2025-05-07T17:32:19.064Z", "id": "LNUnvvrDra89Vhdc9bzdCy", "mean_score": "0.8162764350453172", "stderr": "0.00899565165100951"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-05-07", "model": "mistral-medium-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.32222222222222224, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-medium-2505", "source_row": {"Best score (across scorers)": "0.32222222222222224", "Country": "France", "Model version": "mistral-medium-2505", "Organization": "Mistral AI", "Release date": "2025-05-07", "Started at": "2025-05-07T17:32:21.604Z", "id": "QWw65xeTn8x42SymMsVREN", "mean_score": "0.32222222222222224", "stderr": "0.05965611798482264"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-05-07", "model": "mistral-medium-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5953282828282829, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-medium-2505", "source_row": {"Best score (across scorers)": "0.5953282828282829", "Country": "France", "Model version": "mistral-medium-2505", "Organization": "Mistral AI", "Release date": "2025-05-07", "Started at": "2025-05-07T17:32:22.166Z", "id": "FNMbfvEEm9PRnrJnrqk2PH", "mean_score": "0.5953282828282829", "stderr": "0.02824697229654645"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "mistral-medium-3", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.37689999999999996, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Mistral-Medium-2505", "source_row": {"Model": "Mistral-Medium-2505", "Organization": "Mistral AI", "Overall Acc": "37.69%", "Rank": "48"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "qwen3-4b-instruct-2507", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5222222222222223, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-4b-instruct-2507", "source_row": {"Best score (across scorers)": "0.5222222222222223", "Model version": "qwen3-4b-instruct-2507", "Release date": "2025-08-06", "Started at": "2026-08-28T17:48:22.000Z", "id": "KH7KfZvFc9kZbqiJFdfYTc", "mean_score": "0.5222222222222223", "stderr": "0.06591537955214949"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "qwen3-4b-instruct-2507", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4583333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-4b-instruct-2507", "source_row": {"Best score (across scorers)": "0.4583333333333333", "Model version": "qwen3-4b-instruct-2507", "Release date": "2025-08-06", "Started at": "2026-08-28T17:47:35.000Z", "id": "VhQBa9AViRt2UKSNpEAyjN", "mean_score": "0.4583333333333333", "stderr": "0.028502193235395257"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-4b-instruct-2507", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.3568, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-4B-Instruct-2507 (FC)", "source_row": {"Model": "Qwen3-4B-Instruct-2507 (FC)", "Organization": "Qwen", "Overall Acc": "35.68%", "Rank": "54"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-06-03", "model": "qwen3-235b-a22b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6885699899295065, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-235b-a22b", "source_row": {"Best score (across scorers)": "0.6885699899295065", "Country": "China", "Model version": "qwen3-235b-a22b", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2025-06-03T22:33:05.178Z", "Training compute (FLOP)": "4.752e+24", "id": "BTDXT8mHQu4je5jysTDuXV", "mean_score": "0.6885699899295065", "stderr": "0.009487067376018314"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-05-09", "model": "qwen3-235b-a22b", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.596, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Qwen3 235B A22B diff, no think, Alibaba API", "source_row": {"command": "aider --model openai/qwen3-235b-a22b", "date": "2025-05-09", "dirname": "2025-05-09-17-02-02--qwen3-235b-a22b.unthink_16k_diff", "model": "Qwen3 235B A22B diff, no think, Alibaba API", "pass_rate_2": 59.6, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-06-03", "model": "qwen3-235b-a22b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7070707070707071, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-235b-a22b", "source_row": {"Best score (across scorers)": "0.7070707070707071", "Country": "China", "Model version": "qwen3-235b-a22b", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2025-06-03T11:52:52.449Z", "Training compute (FLOP)": "4.752e+24", "id": "agebmxGseTvmvN4AHN8i4m", "mean_score": "0.7070707070707071", "stderr": "0.027124209393506977"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-01", "model": "deepseek-v3-mar-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.75547583081571, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-V3-0324", "source_row": {"Best score (across scorers)": "0.75547583081571", "Country": "China", "Model version": "DeepSeek-V3-0324", "Organization": "DeepSeek", "Release date": "2025-03-24", "Started at": "2025-04-01T14:52:18.473Z", "Training compute (FLOP)": "3.3e+24", "id": "YFyn28VJ6m2hfop3KGRkBs", "mean_score": "0.75547583081571", "stderr": "0.008603173106577204"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-01", "model": "deepseek-v3-mar-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37777777777777777, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-V3-0324", "source_row": {"Best score (across scorers)": "0.37777777777777777", "Country": "China", "Model version": "DeepSeek-V3-0324", "Organization": "DeepSeek", "Release date": "2025-03-24", "Started at": "2025-04-01T14:33:01.704Z", "Training compute (FLOP)": "3.3e+24", "id": "VpsMexBEhXkPYJBe7v6Fk9", "mean_score": "0.37777777777777777", "stderr": "0.062372203800719374"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-01", "model": "deepseek-v3-mar-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6761363636363636, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-V3-0324", "source_row": {"Best score (across scorers)": "0.6761363636363636", "Country": "China", "Model version": "DeepSeek-V3-0324", "Organization": "DeepSeek", "Release date": "2025-03-24", "Started at": "2025-04-01T14:40:59.363Z", "Training compute (FLOP)": "3.3e+24", "id": "VjfLwKbT6kTprYsMGcz2YT", "mean_score": "0.6761363636363636", "stderr": "0.026836302525030095"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-29", "model": "deepseek-v3-mar-2025", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.646465, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-chat-v3-0324", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "rows": [{"accuracy": "0.650363823580786", "provider_name": "Crusoe", "total_questions": "1145"}, {"accuracy": "0.5230117451776649", "provider_name": "DeepInfra", "total_questions": "985"}, {"accuracy": "0.646465", "provider_name": "GMICloud", "total_questions": "594"}, {"accuracy": "0.1861954", "provider_name": "Novita", "total_questions": "990"}, {"accuracy": "0.66372075", "provider_name": "SiliconFlow", "total_questions": "792"}], "rule": "median over providers (5 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-22", "model": "deepseek-v3-mar-2025", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.4337431393442623, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-chat-v3-0324", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3-0324", "rows": [{"accuracy": "0.4337431393442623", "provider_name": "DeepInfra", "total_questions": "244"}, {"accuracy": "0.4416665", "provider_name": "SiliconFlow", "total_questions": "200"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-03", "model": "qwen3-coder-480b-a35b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.6035354999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-coder-480b-a35b-07-25", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "rows": [{"accuracy": "0.6035354999999999", "provider_name": "Alibaba", "total_questions": "1188"}, {"accuracy": "0.607744", "provider_name": "CoreWeave", "total_questions": "198"}, {"accuracy": "0.6042256518448439", "provider_name": "DeepInfra", "total_questions": "1057"}, {"accuracy": "0.6099860008438818", "provider_name": "Google", "total_questions": "1185"}, {"accuracy": "0.6099888333333333", "provider_name": "Novita", "total_questions": "1188"}, {"accuracy": "0.5962405000000001", "provider_name": "Venice", "total_questions": "1188"}], "rule": "first-party provider (1 of 6 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-03", "model": "qwen3-coder-480b-a35b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.43333342857142854, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen3-coder-480b-a35b-07-25", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen3-coder-480b-a35b-07-25", "rows": [{"accuracy": "0.43333342857142854", "provider_name": "Alibaba", "total_questions": "350"}, {"accuracy": "0.49666650000000007", "provider_name": "CoreWeave", "total_questions": "100"}, {"accuracy": "0.4313334", "provider_name": "DeepInfra", "total_questions": "250"}, {"accuracy": "0.43999974999999997", "provider_name": "Google", "total_questions": "200"}, {"accuracy": "0.4428572857142857", "provider_name": "Novita", "total_questions": "350"}, {"accuracy": "0.4362416845637584", "provider_name": "Venice", "total_questions": "298"}], "rule": "first-party provider (1 of 6 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "o1-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.8164652567975831, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-preview-2024-09-12", "source_row": {"Best score (across scorers)": "0.8164652567975831", "Country": "United States of America", "Model version": "o1-preview-2024-09-12", "Organization": "OpenAI", "Release date": "2024-09-12", "Started at": "2025-01-27T00:00:00.000Z", "id": "8Uu6XsSJCKwcwhXDt8vgzq", "mean_score": "0.8164652567975831", "stderr": "0.010642608258161584"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-07", "model": "o1-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-preview-2024-09-12", "source_row": {"Best score (across scorers)": "0.3111111111111111", "Country": "United States of America", "Model version": "o1-preview-2024-09-12", "Organization": "OpenAI", "Release date": "2024-09-12", "Started at": "2025-03-07T22:14:04.083Z", "id": "Cprjaj9JQ9MY5NCXqeQR5H", "mean_score": "0.3111111111111111", "stderr": "0.0697920592732311"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "o1-preview", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5031565656565656, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "o1-preview-2024-09-12", "source_row": {"Best score (across scorers)": "0.5031565656565656", "Country": "United States of America", "Model version": "o1-preview-2024-09-12", "Organization": "OpenAI", "Release date": "2024-09-12", "Started at": "2025-01-27T00:00:00.000Z", "id": "7CkADTsoNbDd88uqYbqfwj", "mean_score": "0.5031565656565656", "stderr": "0.028257821072909393"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-06", "model": "magistral-small-1-0", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "magistral-small-2506", "source_row": {"Best score (across scorers)": "0.3", "Country": "France", "Model version": "magistral-small-2506", "Organization": "Mistral AI", "Release date": "2025-06-10", "Started at": "2026-08-06T22:17:05.000Z", "id": "hmZPoiHBpqufv9r7ujiK8V", "mean_score": "0.3", "stderr": "0.05924610544247621"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-06", "model": "magistral-small-1-0", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5606060606060606, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "magistral-small-2506", "source_row": {"Best score (across scorers)": "0.5606060606060606", "Country": "France", "Model version": "magistral-small-2506", "Organization": "Mistral AI", "Release date": "2025-06-10", "Started at": "2026-08-06T22:17:01.000Z", "id": "dwGLob8Vhrwx4Wn376jVmt", "mean_score": "0.5606060606060606", "stderr": "0.0353608594752948"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "mistral-small-3-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.30277777777777776, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-3.2-2506", "source_row": {"Best score (across scorers)": "0.30277777777777776", "Country": "France", "Model version": "mistral-small-3.2-2506", "Organization": "Mistral AI", "Release date": "2025-06-20", "Started at": "2026-08-30T22:41:17.000Z", "id": "HyKzRUkV42yTmGrbYQvZxQ", "mean_score": "0.30277777777777776", "stderr": "0.057774499712758075"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "mistral-small-3-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.49053030303030304, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-3.2-2506", "source_row": {"Best score (across scorers)": "0.49053030303030304", "Country": "France", "Model version": "mistral-small-3.2-2506", "Organization": "Mistral AI", "Release date": "2025-06-20", "Started at": "2026-08-28T16:15:28.000Z", "id": "V2CVKXcEdLnEXmppjUEB9M", "mean_score": "0.49053030303030304", "stderr": "0.027717443261958633"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "mistral-small-3-2", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.3715, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Mistral-small-2506 (FC)", "source_row": {"Model": "Mistral-small-2506 (FC)", "Organization": "Mistral AI", "Overall Acc": "37.15%", "Rank": "51"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gemini-1-5-pro-sept-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.7039274924471299, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-pro-002", "source_row": {"Best score (across scorers)": "0.7039274924471299", "Country": "United States of America", "Model version": "gemini-1.5-pro-002", "Organization": "Google DeepMind", "Release date": "2024-09-24", "Started at": "2025-01-27T00:00:00.000Z", "id": "QyqLmgcJpeDMsnQyaaJVkR", "mean_score": "0.7039274924471299", "stderr": "0.010174893415515739"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gemini-1-5-pro-sept-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23055555555555557, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-pro-002", "source_row": {"Best score (across scorers)": "0.23055555555555557", "Country": "United States of America", "Model version": "gemini-1.5-pro-002", "Organization": "Google DeepMind", "Release date": "2024-09-24", "Started at": "2025-02-25T14:05:08.808Z", "id": "fhCdyX9uUS4WPtjLiVdBdK", "mean_score": "0.23055555555555557", "stderr": "0.05186782011216038"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gemini-1-5-pro-sept-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5722853535353535, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-pro-002", "source_row": {"Best score (across scorers)": "0.5722853535353535", "Country": "United States of America", "Model version": "gemini-1.5-pro-002", "Organization": "Google DeepMind", "Release date": "2024-09-24", "Started at": "2025-01-27T00:00:00.000Z", "id": "9TrLEEw8TvsyugyCjwvUvD", "mean_score": "0.5722853535353535", "stderr": "0.027959709636504244"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-22", "model": "gemini-2-0-flash-dec-2024", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.222, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "gemini-2.0-flash-exp", "source_row": {"command": "aider --model gemini/gemini-2.0-flash-exp", "date": "2024-12-22", "dirname": "2024-12-22-20-08-13--gemini-2.0-flash-exp-polyglot-whole", "model": "gemini-2.0-flash-exp", "pass_rate_2": 22.2, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "magistral-small-1-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.28055555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "magistral-small-2509", "source_row": {"Best score (across scorers)": "0.28055555555555556", "Country": "France", "Model version": "magistral-small-2509", "Organization": "Mistral AI", "Release date": "2025-09-18", "Started at": "2026-08-30T22:39:41.000Z", "id": "cSTYZ3FKp5Mbs3xopn68NK", "mean_score": "0.28055555555555556", "stderr": "0.0556407175551872"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "magistral-small-1-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.476010101010101, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "magistral-small-2509", "source_row": {"Best score (across scorers)": "0.476010101010101", "Country": "France", "Model version": "magistral-small-2509", "Organization": "Mistral AI", "Release date": "2025-09-18", "Started at": "2026-08-30T22:39:36.000Z", "id": "3UbYVQELDHAcQgu7sTTnRf", "mean_score": "0.476010101010101", "stderr": "0.025993462698453398"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-01", "model": "qwen2-5-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6718277945619335, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-max-2025-01-25", "source_row": {"Best score (across scorers)": "0.6718277945619335", "Country": "China", "Model version": "qwen-max-2025-01-25", "Organization": "Alibaba", "Release date": "2025-01-25", "Started at": "2025-04-01T17:50:34.087Z", "id": "GFZDNH95TnVegNrEtmUT2Q", "mean_score": "0.6718277945619335", "stderr": "0.011143967442882196"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-01", "model": "qwen2-5-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.16111111111111112, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-max-2025-01-25", "source_row": {"Best score (across scorers)": "0.16111111111111112", "Country": "China", "Model version": "qwen-max-2025-01-25", "Organization": "Alibaba", "Release date": "2025-01-25", "Started at": "2025-04-01T17:11:08.937Z", "id": "PZt4Yb9bpXVPDoPi4uTxwK", "mean_score": "0.16111111111111112", "stderr": "0.04102194808229969"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-01-28", "model": "qwen2-5-max", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.218, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "qwen-max-2025-01-25", "source_row": {"command": "OPENAI_API_BASE=https://dashscope-intl.aliyuncs.com/compatible-mode/v1 aider --model openai/qwen-max-2025-01-25", "date": "2025-01-28", "dirname": "2025-01-28-16-00-03--qwen-max-2025-01-25-polyglot-diff", "model": "qwen-max-2025-01-25", "pass_rate_2": 21.8, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-01", "model": "qwen2-5-max", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5612373737373737, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-max-2025-01-25", "source_row": {"Best score (across scorers)": "0.5612373737373737", "Country": "China", "Model version": "qwen-max-2025-01-25", "Organization": "Alibaba", "Release date": "2025-01-25", "Started at": "2025-04-01T19:22:56.011Z", "id": "GBf7rqHkGGzfX87V37F9mH", "mean_score": "0.5612373737373737", "stderr": "0.027858831475709448"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "deepseek-v3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6485083081570997, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-V3", "source_row": {"Best score (across scorers)": "0.6485083081570997", "Country": "China", "Model version": "DeepSeek-V3", "Organization": "DeepSeek", "Release date": "2024-12-26", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.3e+24", "id": "exvQCLCiga6gE7zvQNp2Z6", "mean_score": "0.6485083081570997", "stderr": "0.010485766934209063"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "deepseek-v3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.15833333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-V3", "source_row": {"Best score (across scorers)": "0.15833333333333333", "Country": "China", "Model version": "DeepSeek-V3", "Organization": "DeepSeek", "Release date": "2024-12-26", "Started at": "2025-02-25T14:05:05.855Z", "Training compute (FLOP)": "3.3e+24", "id": "7saSEhfVvXuz4yTcGy5u2y", "mean_score": "0.15833333333333333", "stderr": "0.0433650088193118"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "deepseek-v3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5653409090909091, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-V3", "source_row": {"Best score (across scorers)": "0.5653409090909091", "Country": "China", "Model version": "DeepSeek-V3", "Organization": "DeepSeek", "Release date": "2024-12-26", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.3e+24", "id": "Vwo3nMA8g2gGBCyTdKFMQJ", "mean_score": "0.5653409090909091", "stderr": "0.02763990029950893"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "deepseek-v3", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.661616, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-chat-v3", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "deepseek/deepseek-chat-v3", "rows": [{"accuracy": "0.535985", "provider_name": "DeepInfra", "total_questions": "792"}, {"accuracy": "0.661616", "provider_name": "Novita", "total_questions": "198"}, {"accuracy": "0.6689113333333334", "provider_name": "StreamLake", "total_questions": "594"}], "rule": "median over providers (3 of 3 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-02", "model": "deepseek-v3", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.4053599698492462, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "deepseek/deepseek-chat-v3", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "deepseek/deepseek-chat-v3", "rows": [{"accuracy": "0.43434354545454545", "provider_name": "DeepInfra", "total_questions": "198"}, {"accuracy": "0.4053599698492462", "provider_name": "StreamLake", "total_questions": "199"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "glm-4-7-flash-none", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.25, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7-flash_none", "source_row": {"Best score (across scorers)": "0.25", "Model version": "glm-4.7-flash_none", "Started at": "2026-08-28T11:17:11.000Z", "id": "fPE4Zajhq5WfccSxEYXXR7", "mean_score": "0.25", "stderr": "0.049460472943941645"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "glm-4-7-flash-none", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4513888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "glm-4.7-flash_none", "source_row": {"Best score (across scorers)": "0.4513888888888889", "Model version": "glm-4.7-flash_none", "Started at": "2026-08-30T22:37:57.000Z", "id": "GSde7u2Mb6GZGpubJjjPam", "mean_score": "0.4513888888888889", "stderr": "0.021751012010888356"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "qwen3-4b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5233585858585859, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-4b", "source_row": {"Best score (across scorers)": "0.5233585858585859", "Country": "China", "Model version": "qwen3-4b", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-28T17:40:19.000Z", "Training compute (FLOP)": "8.64e+23", "id": "G3dinPvwrci2Nu3m9pby2i", "mean_score": "0.5233585858585859", "stderr": "0.030102242301033352"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-31", "model": "phi-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6493580060422961, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "phi-4", "source_row": {"Best score (across scorers)": "0.6493580060422961", "Country": "United States of America", "Model version": "phi-4", "Organization": "Microsoft Research", "Release date": "2024-12-12", "Started at": "2025-01-31T23:48:19.147Z", "Training compute (FLOP)": "9.3202015e+23", "id": "PnvmfAHVxH9oXfcdcUeUZ3", "mean_score": "0.6493580060422961", "stderr": "0.010656871384084826"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "phi-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1375, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "phi-4", "source_row": {"Best score (across scorers)": "0.1375", "Country": "United States of America", "Model version": "phi-4", "Organization": "Microsoft Research", "Release date": "2024-12-12", "Started at": "2025-02-25T13:19:42.221Z", "Training compute (FLOP)": "9.3202015e+23", "id": "5mmrBSQb7hVSJ9VYtNm46i", "mean_score": "0.1375", "stderr": "0.03698083382925635"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-31", "model": "phi-4", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5606060606060606, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "phi-4", "source_row": {"Best score (across scorers)": "0.5606060606060606", "Country": "United States of America", "Model version": "phi-4", "Organization": "Microsoft Research", "Release date": "2024-12-12", "Started at": "2025-01-31T23:03:31.781Z", "Training compute (FLOP)": "9.3202015e+23", "id": "RAwuPRwJedy7P2KeQhEmr8", "mean_score": "0.5606060606060606", "stderr": "0.02590274664132679"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "phi-4", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.2879, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Phi-4 (Prompt)", "source_row": {"Model": "Phi-4 (Prompt)", "Organization": "Microsoft", "Overall Acc": "28.79%", "Rank": "70"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-04-07", "model": "qwen-plus-jan-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6527567975830816, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-plus-2025-01-25", "source_row": {"Best score (across scorers)": "0.6527567975830816", "Country": "China", "Model version": "qwen-plus-2025-01-25", "Organization": "Alibaba", "Release date": "2025-01-25", "Started at": "2025-04-07T17:38:39.334Z", "id": "J3WoEpLKAJZWGGpyBVCCCn", "mean_score": "0.6527567975830816", "stderr": "0.011334654774046507"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-07", "model": "qwen-plus-jan-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17777777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-plus-2025-01-25", "source_row": {"Best score (across scorers)": "0.17777777777777778", "Country": "China", "Model version": "qwen-plus-2025-01-25", "Organization": "Alibaba", "Release date": "2025-01-25", "Started at": "2025-04-07T19:09:43.970Z", "id": "AULNJG2C75JBP6uaiAgj3P", "mean_score": "0.17777777777777778", "stderr": "0.04360797453817489"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-07", "model": "qwen-plus-jan-2025", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4810606060606061, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-plus-2025-01-25", "source_row": {"Best score (across scorers)": "0.4810606060606061", "Country": "China", "Model version": "qwen-plus-2025-01-25", "Organization": "Alibaba", "Release date": "2025-01-25", "Started at": "2025-04-07T19:28:06.394Z", "id": "QtraMoKiQRRpZc9txpFAYF", "mean_score": "0.4810606060606061", "stderr": "0.027144936812277986"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "claude-3-5-sonnet-october-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5694864048338368, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-sonnet-20241022", "source_row": {"Best score (across scorers)": "0.5694864048338368", "Country": "United States of America", "Model version": "claude-3-5-sonnet-20241022", "Organization": "Anthropic", "Release date": "2024-10-22", "Started at": "2025-01-27T00:00:00.000Z", "id": "AjroyuZECXLtMTwnzaMHtX", "mean_score": "0.5694864048338368", "stderr": "0.011091284283577038"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "claude-3-5-sonnet-october-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08472222222222223, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-sonnet-20241022", "source_row": {"Best score (across scorers)": "0.08472222222222223", "Country": "United States of America", "Model version": "claude-3-5-sonnet-20241022", "Organization": "Anthropic", "Release date": "2024-10-22", "Started at": "2025-02-25T14:05:03.625Z", "id": "geVVaRWxuprKUqnW2VzvCa", "mean_score": "0.08472222222222223", "stderr": "0.026363956855902478"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-01-17", "model": "claude-3-5-sonnet-october-2024", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.516, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "claude-3-5-sonnet-20241022", "source_row": {"command": "aider --model claude-3-5-sonnet-20241022", "date": "2025-01-17", "dirname": "2025-01-17-19-44-33--sonnet-baseline-jan-17", "model": "claude-3-5-sonnet-20241022", "pass_rate_2": 51.6, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "claude-3-5-sonnet-october-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.553030303030303, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-sonnet-20241022", "source_row": {"Best score (across scorers)": "0.553030303030303", "Country": "United States of America", "Model version": "claude-3-5-sonnet-20241022", "Organization": "Anthropic", "Release date": "2024-10-22", "Started at": "2025-01-27T00:00:00.000Z", "id": "JCCVcVLz3J4TQXSU67hMJZ", "mean_score": "0.553030303030303", "stderr": "0.028208090147078847"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-17", "model": "grok-2-dec-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6351963746223565, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-2-1212", "source_row": {"Best score (across scorers)": "0.6351963746223565", "Country": "United States of America", "Model version": "grok-2-1212", "Organization": "xAI", "Release date": "2024-12-12", "Started at": "2025-02-17T21:39:16.270Z", "Training compute (FLOP)": "2.96e+25", "id": "Kpck3pLw3veM5Mt7WZAb8G", "mean_score": "0.6351963746223565", "stderr": "0.01323438169590836"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "grok-2-dec-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.11527777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-2-1212", "source_row": {"Best score (across scorers)": "0.11527777777777778", "Country": "United States of America", "Model version": "grok-2-1212", "Organization": "xAI", "Release date": "2024-12-12", "Started at": "2025-02-25T14:38:30.971Z", "Training compute (FLOP)": "2.96e+25", "id": "PpSVDb67Ucbxpcd8JQv5ii", "mean_score": "0.11527777777777778", "stderr": "0.03382424700930314"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "grok-2-dec-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5378787878787878, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "grok-2-1212", "source_row": {"Best score (across scorers)": "0.5378787878787878", "Country": "United States of America", "Model version": "grok-2-1212", "Organization": "xAI", "Release date": "2024-12-12", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.96e+25", "id": "83DRJD4FvjwqU9eStRaNiU", "mean_score": "0.5378787878787878", "stderr": "0.027066342316607196"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-26", "model": "qwen2-5-coder-32b-instruct", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.16399999999999998, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "Qwen2.5-Coder-32B-Instruct", "source_row": {"command": "aider --model openai/Qwen2.5-Coder-32B-Instruct", "date": "2024-12-26", "dirname": "2024-12-26-00-55-20--Qwen2.5-Coder-32B-Instruct", "model": "Qwen2.5-Coder-32B-Instruct", "pass_rate_2": 16.4, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gemini-1-5-flash-sep-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6186744712990937, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-002", "source_row": {"Best score (across scorers)": "0.6186744712990937", "Country": "United States of America", "Model version": "gemini-1.5-flash-002", "Organization": "Google DeepMind", "Release date": "2024-09-24", "Started at": "2025-01-27T00:00:00.000Z", "id": "kDnfjYKQpnLGEjuhBc9AvM", "mean_score": "0.6186744712990937", "stderr": "0.01120122127580226"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gemini-1-5-flash-sep-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1625, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-002", "source_row": {"Best score (across scorers)": "0.1625", "Country": "United States of America", "Model version": "gemini-1.5-flash-002", "Organization": "Google DeepMind", "Release date": "2024-09-24", "Started at": "2025-02-25T14:05:07.974Z", "id": "QdGZ2m52PWt3EkLPbcoRwz", "mean_score": "0.1625", "stderr": "0.04111375571619442"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gemini-1-5-flash-sep-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.47316919191919193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-002", "source_row": {"Best score (across scorers)": "0.47316919191919193", "Country": "United States of America", "Model version": "gemini-1.5-flash-002", "Organization": "Google DeepMind", "Release date": "2024-09-24", "Started at": "2025-01-27T00:00:00.000Z", "id": "PCLvrs4TEBtqQqcYeU8pAG", "mean_score": "0.47316919191919193", "stderr": "0.027737408277040958"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-08", "model": "llama-4-scout", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.6227341389728097, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-4-Scout-17B-16E-Instruct", "source_row": {"Best score (across scorers)": "0.6227341389728097", "Country": "United States of America", "Model version": "Llama-4-Scout-17B-16E-Instruct", "Organization": "Meta AI", "Release date": "2025-04-05", "Started at": "2025-04-08T09:22:06.630Z", "Training compute (FLOP)": "4.08e+24", "id": "c6rTyTFCeoHMBzC2QwjxFL", "mean_score": "0.6227341389728097", "stderr": "0.012022405873783983"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-08", "model": "llama-4-scout", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.07777777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-4-Scout-17B-16E-Instruct", "source_row": {"Best score (across scorers)": "0.07777777777777778", "Country": "United States of America", "Model version": "Llama-4-Scout-17B-16E-Instruct", "Organization": "Meta AI", "Release date": "2025-04-05", "Started at": "2025-04-08T09:43:28.090Z", "Training compute (FLOP)": "4.08e+24", "id": "T3X5DeC47iqCaoRYXu5k8L", "mean_score": "0.07777777777777778", "stderr": "0.030069597533844094"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-08", "model": "llama-4-scout", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5183080808080808, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-4-Scout-17B-16E-Instruct", "source_row": {"Best score (across scorers)": "0.5183080808080808", "Country": "United States of America", "Model version": "Llama-4-Scout-17B-16E-Instruct", "Organization": "Meta AI", "Release date": "2025-04-05", "Started at": "2025-04-08T08:34:50.470Z", "Training compute (FLOP)": "4.08e+24", "id": "MsXAXDEKP3xBKtiA7aTBBG", "mean_score": "0.5183080808080808", "stderr": "0.031549020124994755"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "llama-4-scout", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.2813, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Llama-4-Scout-17B-16E-Instruct (FC)", "source_row": {"Model": "Llama-4-Scout-17B-16E-Instruct (FC)", "Organization": "Meta", "Overall Acc": "28.13%", "Rank": "72"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "gemma-3-12b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.16666666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-12b-it", "source_row": {"Best score (across scorers)": "0.16666666666666666", "Country": "United States of America", "Model version": "gemma-3-12b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-28T08:22:55.000Z", "Training compute (FLOP)": "8.64e+23", "id": "CAyLyhDHZiMQBZDB3qd8tG", "mean_score": "0.16666666666666666", "stderr": "0.0424174778156649"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "gemma-3-12b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.39457070707070707, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-12b-it", "source_row": {"Best score (across scorers)": "0.39457070707070707", "Country": "United States of America", "Model version": "gemma-3-12b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-28T08:22:17.000Z", "Training compute (FLOP)": "8.64e+23", "id": "KfQKr2ibCFMx5xBELoiM3Y", "mean_score": "0.39457070707070707", "stderr": "0.025254327795483048"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-07", "model": "gemma-3-12b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.37878799999999996, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemma-3-12b-it", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-12b-it", "rows": [{"accuracy": "0.405724", "provider_name": "DeepInfra", "total_questions": "198"}, {"accuracy": "0.37878799999999996", "provider_name": "NextBit", "total_questions": "396"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gemma-3-12b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.3043, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Gemma-3-12b-it (Prompt)", "source_row": {"Model": "Gemma-3-12b-it (Prompt)", "Organization": "Google", "Overall Acc": "30.43%", "Rank": "66"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-10-08", "model": "gemma-3-12b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.48, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemma-3-12b-it", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "google/gemma-3-12b-it", "rows": [{"accuracy": "0.48", "provider_name": "DeepInfra", "total_questions": "50"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "deepseek-r1-distill-qwen-1-5b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.21388888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Qwen-1.5B", "source_row": {"Best score (across scorers)": "0.21388888888888888", "Country": "China", "Model version": "DeepSeek-R1-Distill-Qwen-1.5B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2026-08-28T07:53:04.000Z", "id": "Gs234P3jzGiLm68KjyzYZL", "mean_score": "0.21388888888888888", "stderr": "0.05279771021800918"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "deepseek-r1-distill-qwen-1-5b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.33585858585858586, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "DeepSeek-R1-Distill-Qwen-1.5B", "source_row": {"Best score (across scorers)": "0.33585858585858586", "Country": "China", "Model version": "DeepSeek-R1-Distill-Qwen-1.5B", "Organization": "DeepSeek", "Release date": "2025-01-20", "Started at": "2026-08-28T07:52:16.000Z", "id": "ESnXA8LqMqhjcsoWQaMLVQ", "mean_score": "0.33585858585858586", "stderr": "0.020592418168039042"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "qwen2-5-72b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.631703172205438, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-72b-instruct", "source_row": {"Best score (across scorers)": "0.631703172205438", "Country": "China", "Model version": "qwen2.5-72b-instruct", "Organization": "Alibaba", "Release date": "2024-09-19", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.8e+24", "id": "ni4DJboYYpdRssVeAuro2V", "mean_score": "0.631703172205438", "stderr": "0.01071424933114761"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "qwen2-5-72b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08055555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-72b-instruct", "source_row": {"Best score (across scorers)": "0.08055555555555556", "Country": "China", "Model version": "qwen2.5-72b-instruct", "Organization": "Alibaba", "Release date": "2024-09-19", "Started at": "2025-02-25T14:05:03.625Z", "Training compute (FLOP)": "7.8e+24", "id": "67YbVQfJh7sd77KH3zFgTn", "mean_score": "0.08055555555555556", "stderr": "0.026098213018967212"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "qwen2-5-72b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4914772727272727, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-72b-instruct", "source_row": {"Best score (across scorers)": "0.4914772727272727", "Country": "China", "Model version": "qwen2.5-72b-instruct", "Organization": "Alibaba", "Release date": "2024-09-19", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.8e+24", "id": "WtALgW6VSFdihDZWXRm2AZ", "mean_score": "0.4914772727272727", "stderr": "0.026947767404178667"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-01", "model": "qwen2-5-72b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.461279, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen-2.5-72b-instruct", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "rows": [{"accuracy": "0.4765793529411765", "provider_name": "DeepInfra", "total_questions": "306"}, {"accuracy": "0.461279", "provider_name": "Novita", "total_questions": "198"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-09-01", "model": "qwen2-5-72b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.25, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen-2.5-72b-instruct", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-72b-instruct", "rows": [{"accuracy": "0.25", "provider_name": "DeepInfra", "total_questions": "50"}, {"accuracy": "0.2622223333333333", "provider_name": "Novita", "total_questions": "150"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-02-25", "model": "mistral-large-2-nov-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5028323262839879, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2411", "source_row": {"Best score (across scorers)": "0.5028323262839879", "Country": "France", "Model version": "mistral-large-2411", "Organization": "Mistral AI", "Release date": "2024-11-18", "Started at": "2025-02-25T12:31:34.494Z", "id": "MMqL6fQSwsBKAdBSfxrYpU", "mean_score": "0.5028323262839879", "stderr": "0.011469451623480956"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "mistral-large-2-nov-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.07777777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2411", "source_row": {"Best score (across scorers)": "0.07777777777777778", "Country": "France", "Model version": "mistral-large-2411", "Organization": "Mistral AI", "Release date": "2024-11-18", "Started at": "2025-02-25T14:05:06.293Z", "id": "beRGgJXTJggTi8PPEZNoZ5", "mean_score": "0.07777777777777778", "stderr": "0.024662778950157508"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-25", "model": "mistral-large-2-nov-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5132575757575758, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2411", "source_row": {"Best score (across scorers)": "0.5132575757575758", "Country": "France", "Model version": "mistral-large-2411", "Organization": "Mistral AI", "Release date": "2024-11-18", "Started at": "2025-02-25T12:31:34.069Z", "id": "n9gpXfUzo8SbWqvrG2q4xM", "mean_score": "0.5132575757575758", "stderr": "0.026887818295509087"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "mistral-large-2-nov-2024", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.3837, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "mistral-large-2411 (FC)", "source_row": {"Model": "mistral-large-2411 (FC)", "Organization": "Mistral AI", "Overall Acc": "38.37%", "Rank": "46"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-23", "model": "yi-lightning", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.129, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "yi-lightning", "source_row": {"command": "aider --model openai/yi-lightning", "date": "2024-12-23", "dirname": "2024-12-23-01-11-56--yi-test", "model": "yi-lightning", "pass_rate_2": 12.9, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "claude-3-5-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5168051359516617, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-sonnet-20240620", "source_row": {"Best score (across scorers)": "0.5168051359516617", "Country": "United States of America", "Model version": "claude-3-5-sonnet-20240620", "Organization": "Anthropic", "Release date": "2024-06-20", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.700000000000001e+25", "id": "H42HJG29bwqzaMewTMsmky", "mean_score": "0.5168051359516617", "stderr": "0.0116250120233614"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "claude-3-5-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06527777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-sonnet-20240620", "source_row": {"Best score (across scorers)": "0.06527777777777778", "Country": "United States of America", "Model version": "claude-3-5-sonnet-20240620", "Organization": "Anthropic", "Release date": "2024-06-20", "Started at": "2025-02-25T14:05:09.497Z", "Training compute (FLOP)": "2.700000000000001e+25", "id": "2xNPhKLsiWLHQBLLDGU4VJ", "mean_score": "0.06527777777777778", "stderr": "0.022293131815660016"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "claude-3-5-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5404040404040404, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-sonnet-20240620", "source_row": {"Best score (across scorers)": "0.5404040404040404", "Country": "United States of America", "Model version": "claude-3-5-sonnet-20240620", "Organization": "Anthropic", "Release date": "2024-06-20", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.700000000000001e+25", "id": "78Wpvp8WDByAcQRBas72cs", "mean_score": "0.5404040404040404", "stderr": "0.027528555453348653"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-30", "model": "qwen2-5-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5607061933534743, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-32b-instruct", "source_row": {"Best score (across scorers)": "0.5607061933534743", "Country": "China", "Model version": "qwen2.5-32b-instruct", "Organization": "Alibaba", "Release date": "2024-09-17", "Started at": "2025-01-30T19:02:48.179Z", "Training compute (FLOP)": "3.51e+24", "id": "Nk4mmCXFH3mEyCYnQPdPx5", "mean_score": "0.5607061933534743", "stderr": "0.01076720798872177"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "qwen2-5-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.07361111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-32b-instruct", "source_row": {"Best score (across scorers)": "0.07361111111111111", "Country": "China", "Model version": "qwen2.5-32b-instruct", "Organization": "Alibaba", "Release date": "2024-09-17", "Started at": "2025-02-25T14:05:07.648Z", "Training compute (FLOP)": "3.51e+24", "id": "NRCNnCCL5aQKsZf6cKC69Z", "mean_score": "0.07361111111111111", "stderr": "0.021420595022864277"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-30", "model": "qwen2-5-32b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46085858585858586, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-32b-instruct", "source_row": {"Best score (across scorers)": "0.46085858585858586", "Country": "China", "Model version": "qwen2.5-32b-instruct", "Organization": "Alibaba", "Release date": "2024-09-17", "Started at": "2025-01-30T18:31:31.252Z", "Training compute (FLOP)": "3.51e+24", "id": "d6JihAFVqUy3u9suNt4P3L", "mean_score": "0.46085858585858586", "stderr": "0.02595783004579065"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gpt-4o-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5104796072507553, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-05-13", "source_row": {"Best score (across scorers)": "0.5104796072507553", "Country": "United States of America", "Model version": "gpt-4o-2024-05-13", "Organization": "OpenAI", "Release date": "2024-05-13", "Started at": "2025-01-27T00:00:00.000Z", "id": "Uqsvq9TAGoqD2xpKfhHBmS", "mean_score": "0.5104796072507553", "stderr": "0.01120841984218505"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gpt-4o-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0625, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-05-13", "source_row": {"Best score (across scorers)": "0.0625", "Country": "United States of America", "Model version": "gpt-4o-2024-05-13", "Organization": "OpenAI", "Release date": "2024-05-13", "Started at": "2025-02-25T14:05:03.623Z", "id": "9cCn5aW522cTRxJSE69crm", "mean_score": "0.0625", "stderr": "0.023249986423369555"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gpt-4o-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4889520202020202, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4o-2024-05-13", "source_row": {"Best score (across scorers)": "0.4889520202020202", "Country": "United States of America", "Model version": "gpt-4o-2024-05-13", "Organization": "OpenAI", "Release date": "2024-05-13", "Started at": "2025-01-27T00:00:00.000Z", "id": "Y59q8WbEDp2HH8oovyJCj9", "mean_score": "0.4889520202020202", "stderr": "0.026254109873371994"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "gpt-4o-may-2024", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.542088, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4o-2024-05-13", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "openai/gpt-4o-2024-05-13", "rows": [{"accuracy": "0.505051", "provider_name": "Azure", "total_questions": "198"}, {"accuracy": "0.542088", "provider_name": "OpenAI", "total_questions": "198"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "gpt-4o-may-2024", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.44, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "openai/gpt-4o-2024-05-13", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "openai/gpt-4o-2024-05-13", "rows": [{"accuracy": "0.36666699999999997", "provider_name": "Azure", "total_questions": "50"}, {"accuracy": "0.44", "provider_name": "OpenAI", "total_questions": "50"}], "rule": "first-party provider (1 of 2 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "llama-3-1-405b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.49773413897280966, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-405B-Instruct", "source_row": {"Best score (across scorers)": "0.49773413897280966", "Country": "United States of America", "Model version": "Llama-3.1-405B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.8e+25", "id": "ZR7amdUBRr5Cke8Aoz4snr", "mean_score": "0.49773413897280966", "stderr": "0.012185275936569536"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "llama-3-1-405b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.09722222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-405B-Instruct", "source_row": {"Best score (across scorers)": "0.09722222222222222", "Country": "United States of America", "Model version": "Llama-3.1-405B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2025-02-25T14:05:05.407Z", "Training compute (FLOP)": "3.8e+25", "id": "2fQJP3brRApfBdQq7Zy3bm", "mean_score": "0.09722222222222222", "stderr": "0.03218644609162339"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "llama-3-1-405b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5091540404040404, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-405B-Instruct", "source_row": {"Best score (across scorers)": "0.5091540404040404", "Country": "United States of America", "Model version": "Llama-3.1-405B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.8e+25", "id": "S5QYXSvQBRSbUbXSnAGbMm", "mean_score": "0.5091540404040404", "stderr": "0.025861992702770648"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-04-07", "model": "qwen-turbo", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.5623111782477341, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-turbo-2024-11-01", "source_row": {"Best score (across scorers)": "0.5623111782477341", "Country": "China", "Model version": "qwen-turbo-2024-11-01", "Organization": "Alibaba", "Release date": "2024-11-01", "Started at": "2025-04-07T19:55:58.553Z", "id": "jWV936hYKg6rYcPEynzEEt", "mean_score": "0.5623111782477341", "stderr": "0.011364630947550552"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-07", "model": "qwen-turbo", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06111111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-turbo-2024-11-01", "source_row": {"Best score (across scorers)": "0.06111111111111111", "Country": "China", "Model version": "qwen-turbo-2024-11-01", "Organization": "Alibaba", "Release date": "2024-11-01", "Started at": "2025-04-07T19:18:53.147Z", "id": "G5gxbTH33S7LXA29o5fgmv", "mean_score": "0.06111111111111111", "stderr": "0.022340279838327228"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-04-07", "model": "qwen-turbo", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.41792929292929293, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen-turbo-2024-11-01", "source_row": {"Best score (across scorers)": "0.41792929292929293", "Country": "China", "Model version": "qwen-turbo-2024-11-01", "Organization": "Alibaba", "Release date": "2024-11-01", "Started at": "2025-04-07T21:38:51.185Z", "id": "9Ke5At9bYuf56SmKJJ7Sz7", "mean_score": "0.41792929292929293", "stderr": "0.02617727508870251"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "gpt-4-turbo-apr-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.007017543859649123, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-turbo-2024-04-09", "source_row": {"Best score (across scorers)": "0.007017543859649123", "Country": "United States of America", "Model version": "gpt-4-turbo-2024-04-09", "Organization": "OpenAI", "Release date": "2024-04-09", "Started at": "2026-08-27T17:54:49.000Z", "id": "bC2dbN4x2GWWmnnfseKxgQ", "mean_score": "0.007017543859649123", "stderr": "0.004953408961963604"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-02-27", "model": "gpt-4-turbo-apr-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46733383685800606, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-turbo-2024-04-09", "source_row": {"Best score (across scorers)": "0.46733383685800606", "Country": "United States of America", "Model version": "gpt-4-turbo-2024-04-09", "Organization": "OpenAI", "Release date": "2024-04-09", "Started at": "2025-02-27T14:29:46.292Z", "id": "AZAJomaSEfZtqYqXUre9fS", "mean_score": "0.46733383685800606", "stderr": "0.01142825893039598"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-27", "model": "gpt-4-turbo-apr-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-turbo-2024-04-09", "source_row": {"Best score (across scorers)": "0.06666666666666667", "Country": "United States of America", "Model version": "gpt-4-turbo-2024-04-09", "Organization": "OpenAI", "Release date": "2024-04-09", "Started at": "2025-02-27T14:29:44.767Z", "id": "MZABVPEY8VCPAyCZ4iir68", "mean_score": "0.06666666666666667", "stderr": "0.024034381938205007"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-02-27", "model": "gpt-4-turbo-apr-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4659090909090909, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-turbo-2024-04-09", "source_row": {"Best score (across scorers)": "0.4659090909090909", "Country": "United States of America", "Model version": "gpt-4-turbo-2024-04-09", "Organization": "OpenAI", "Release date": "2024-04-09", "Started at": "2025-02-27T14:29:45.537Z", "id": "TaCoN7eyPQEfDWXqs2MpcF", "mean_score": "0.4659090909090909", "stderr": "0.026576883332580552"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "mistral-large-2-jul-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4481684290030212, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2407", "source_row": {"Best score (across scorers)": "0.4481684290030212", "Country": "France", "Model version": "mistral-large-2407", "Organization": "Mistral AI", "Release date": "2024-07-24", "Started at": "2025-01-27T00:00:00.000Z", "id": "div9uE9eovqykqx883kSpE", "mean_score": "0.4481684290030212", "stderr": "0.010703488686415494"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "mistral-large-2-jul-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08472222222222223, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2407", "source_row": {"Best score (across scorers)": "0.08472222222222223", "Country": "France", "Model version": "mistral-large-2407", "Organization": "Mistral AI", "Release date": "2024-07-24", "Started at": "2025-02-25T11:59:42.758Z", "id": "bHLfPfZj8HgD9Gef75NtrM", "mean_score": "0.08472222222222223", "stderr": "0.024742642797709363"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "mistral-large-2-jul-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.49021464646464646, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2407", "source_row": {"Best score (across scorers)": "0.49021464646464646", "Country": "France", "Model version": "mistral-large-2407", "Organization": "Mistral AI", "Release date": "2024-07-24", "Started at": "2025-01-27T00:00:00.000Z", "id": "ZQPNnicTuhuscJvni85PFY", "mean_score": "0.49021464646464646", "stderr": "0.025918840988656967"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-03-18", "model": "mistral-small-3-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4677114803625378, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-2503", "source_row": {"Best score (across scorers)": "0.4677114803625378", "Country": "France", "Model version": "mistral-small-2503", "Organization": "Mistral AI", "Release date": "2025-03-17", "Started at": "2025-03-18T16:08:03.120Z", "id": "Kas7Y5PBsT7Li5Wm6Y8mj8", "mean_score": "0.4677114803625378", "stderr": "0.011995700377965737"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-18", "model": "mistral-small-3-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.058333333333333334, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-2503", "source_row": {"Best score (across scorers)": "0.058333333333333334", "Country": "France", "Model version": "mistral-small-2503", "Organization": "Mistral AI", "Release date": "2025-03-17", "Started at": "2025-03-18T16:08:02.049Z", "id": "JxqtHWrwCtYoQYBB3jVqB9", "mean_score": "0.058333333333333334", "stderr": "0.0226830377599695"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-03-18", "model": "mistral-small-3-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.47474747474747475, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-2503", "source_row": {"Best score (across scorers)": "0.47474747474747475", "Country": "France", "Model version": "mistral-small-2503", "Organization": "Mistral AI", "Release date": "2025-03-17", "Started at": "2025-03-18T16:08:06.162Z", "id": "S8bBDAbucsFceirHxeEDvu", "mean_score": "0.47474747474747475", "stderr": "0.027616495140012243"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2025-04-19", "model": "openhands-lm-32b-v0-1", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.102, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "openhands-lm-32b-v0.1", "source_row": {"command": "aider --model openrouter/all-hands/openhands-lm-32b-v0.1", "date": "2025-04-19", "dirname": "2025-04-19-14-43-04--o4-mini-patch", "model": "openhands-lm-32b-v0.1", "pass_rate_2": 10.2, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "math_level_5", "eval_date": "2025-01-30", "model": "mistral-small-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4481684290030212, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-2501", "source_row": {"Best score (across scorers)": "0.4481684290030212", "Country": "France", "Model version": "mistral-small-2501", "Organization": "Mistral AI", "Release date": "2025-01-25", "Started at": "2025-01-30T17:26:30.440Z", "Training compute (FLOP)": "1.152e+24", "id": "NbsnvBsMoMizozbPZY8LLb", "mean_score": "0.4481684290030212", "stderr": "0.010854927233993669"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "mistral-small-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06666666666666667, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-3-2501", "source_row": {"Best score (across scorers)": "0.06666666666666667", "Country": "France", "Model version": "mistral-small-3-2501", "Organization": "Mistral AI", "Release date": "2025-01-30", "Started at": "2026-08-30T22:40:34.000Z", "Training compute (FLOP)": "1.152e+24", "id": "LEBmfabfasGXFwRwhN9j35", "mean_score": "0.06666666666666667", "stderr": "0.02682716849943299"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "mistral-small-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.47285353535353536, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-small-3-2501", "source_row": {"Best score (across scorers)": "0.47285353535353536", "Country": "France", "Model version": "mistral-small-3-2501", "Organization": "Mistral AI", "Release date": "2025-01-30", "Started at": "2026-08-30T22:40:23.000Z", "Training compute (FLOP)": "1.152e+24", "id": "YJPwhEJa6qQWB23HQjByxt", "mean_score": "0.47285353535353536", "stderr": "0.027888880246470463"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-03-12", "model": "claude-3-5-haiku", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.46355740181268884, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-haiku-20241022", "source_row": {"Best score (across scorers)": "0.46355740181268884", "Country": "United States of America", "Model version": "claude-3-5-haiku-20241022", "Organization": "Anthropic", "Release date": "2024-10-22", "Started at": "2025-03-12T23:51:45.093Z", "id": "GSvi9Zf5aK3t5TfCPqNRHy", "mean_score": "0.46355740181268884", "stderr": "0.011611728105526228"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "claude-3-5-haiku", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.043055555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-haiku-20241022", "source_row": {"Best score (across scorers)": "0.043055555555555555", "Country": "United States of America", "Model version": "claude-3-5-haiku-20241022", "Organization": "Anthropic", "Release date": "2024-10-22", "Started at": "2025-02-25T14:05:03.257Z", "id": "cqhrR5KnwpcuAVpHkZGQFd", "mean_score": "0.043055555555555555", "stderr": "0.016237638827815495"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "aider_polyglot", "eval_date": "2024-12-21", "model": "claude-3-5-haiku", "retrieved_at": "2026-10-18T12:17:28Z", "score": 0.28, "source": "Aider LLM Leaderboards", "source_licence": "Apache-2.0", "source_model_name": "claude-3-5-haiku-20241022", "source_row": {"command": "aider --model claude-3-5-haiku-20241022", "date": "2024-12-21", "dirname": "2024-12-21-21-46-27--polyglot-haiku-diff", "model": "claude-3-5-haiku-20241022", "pass_rate_2": 28.0, "test_cases": 225}, "source_sha256": "85a50b25953512d18ba4bb0c23c0b8e626fcf9a5b52d287644b8a0b44b9535de", "source_snapshot": "https://raw.githubusercontent.com/Aider-AI/aider/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/website/_data/polyglot_leaderboard.yml"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-03-12", "model": "claude-3-5-haiku", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3813131313131313, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-5-haiku-20241022", "source_row": {"Best score (across scorers)": "0.3813131313131313", "Country": "United States of America", "Model version": "claude-3-5-haiku-20241022", "Organization": "Anthropic", "Release date": "2024-10-22", "Started at": "2025-03-12T23:51:44.119Z", "id": "JpLxrpvthx7A75sbaHZ9Et", "mean_score": "0.3813131313131313", "stderr": "0.026351515390511556"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gemini-1-5-pro-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4074773413897281, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-pro-001", "source_row": {"Best score (across scorers)": "0.4074773413897281", "Country": "United States of America", "Model version": "gemini-1.5-pro-001", "Organization": "Google DeepMind", "Release date": "2024-05-14", "Started at": "2025-01-27T00:00:00.000Z", "id": "PxLmyq7PFfARAuqxg5oPtE", "mean_score": "0.4074773413897281", "stderr": "0.010574902459145802"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gemini-1-5-pro-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.06805555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-pro-001", "source_row": {"Best score (across scorers)": "0.06805555555555555", "Country": "United States of America", "Model version": "gemini-1.5-pro-001", "Organization": "Google DeepMind", "Release date": "2024-05-14", "Started at": "2025-02-25T14:38:32.379Z", "id": "3S6fjGAMKWKK3zcdxkmh9H", "mean_score": "0.06805555555555555", "stderr": "0.01893034295514878"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gemini-1-5-pro-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4586489898989899, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-pro-001", "source_row": {"Best score (across scorers)": "0.4586489898989899", "Country": "United States of America", "Model version": "gemini-1.5-pro-001", "Organization": "Google DeepMind", "Release date": "2024-05-14", "Started at": "2025-01-27T00:00:00.000Z", "id": "PKdqpKZ9tqDWyWJ7GksBWz", "mean_score": "0.4586489898989899", "stderr": "0.02617295639558329"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "chess_puzzles", "eval_date": "2026-10-13", "model": "claude-3-opus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.03, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-opus-20240229", "source_row": {"Best score (across scorers)": "0.03", "Country": "United States of America", "Model version": "claude-3-opus-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2026-10-13T13:45:40.000Z", "id": "WAbi8sZpyNAZoNN3Gmg22y", "mean_score": "0.03", "stderr": "0.01714466079977653"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#chess_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "claude-3-opus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.37481117824773413, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-opus-20240229", "source_row": {"Best score (across scorers)": "0.37481117824773413", "Country": "United States of America", "Model version": "claude-3-opus-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2025-01-27T00:00:00.000Z", "id": "6SNzvpd6kXXxqpJoaHsSDd", "mean_score": "0.37481117824773413", "stderr": "0.01058728426376393"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "claude-3-opus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.04722222222222222, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-opus-20240229", "source_row": {"Best score (across scorers)": "0.04722222222222222", "Country": "United States of America", "Model version": "claude-3-opus-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2025-02-25T14:05:00.461Z", "id": "mYb9MGQD39RS8VdDyMqujr", "mean_score": "0.04722222222222222", "stderr": "0.019579258477123922"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "claude-3-opus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4715909090909091, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-opus-20240229", "source_row": {"Best score (across scorers)": "0.4715909090909091", "Country": "United States of America", "Model version": "claude-3-opus-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2025-01-27T00:00:00.000Z", "id": "hTnDa4gn8RYduCEYqoNywe", "mean_score": "0.4715909090909091", "stderr": "0.026395393956533515"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "simpleqa_verified", "eval_date": "2026-08-31", "model": "claude-3-opus", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.126, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-opus-20240229", "source_row": {"Best score (across scorers)": "0.126", "Country": "United States of America", "Model version": "claude-3-opus-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2026-08-31T18:18:52.000Z", "id": "NxVp4VF8uQbL5S6hx6HFRB", "mean_score": "0.126", "stderr": "0.010499249222407964"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#simpleqa_verified.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "qwen3-1-7b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.08055555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-1.7b_none", "source_row": {"Best score (across scorers)": "0.08055555555555556", "Country": "China", "Model version": "qwen3-1.7b_none", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-28T16:47:54.000Z", "Training compute (FLOP)": "3.672e+23", "id": "bbqokudairnKtbyjBjuWCx", "mean_score": "0.08055555555555556", "stderr": "0.0281353251405273"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "qwen3-1-7b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.380050505050505, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen3-1.7b", "source_row": {"Best score (across scorers)": "0.380050505050505", "Country": "China", "Model version": "qwen3-1.7b", "Organization": "Alibaba", "Release date": "2025-04-29", "Started at": "2026-08-28T16:47:04.000Z", "Training compute (FLOP)": "3.672e+23", "id": "F3CtjUceBrMi9iYmjULx7J", "mean_score": "0.380050505050505", "stderr": "0.02748387280745701"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-1-7b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.2841, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-1.7B (FC)", "source_row": {"Model": "Qwen3-1.7B (FC)", "Organization": "Qwen", "Overall Acc": "28.41%", "Rank": "71"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "qwen3-0-6b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.23929999999999998, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Qwen3-0.6B (FC)", "source_row": {"Model": "Qwen3-0.6B (FC)", "Organization": "Qwen", "Overall Acc": "23.93%", "Rank": "92"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "qwen2-72b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.39067220543806647, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2-72b-instruct", "source_row": {"Best score (across scorers)": "0.39067220543806647", "Country": "China", "Model version": "qwen2-72b-instruct", "Organization": "Alibaba", "Release date": "2024-06-07", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.02e+24", "id": "9mWoY9fM2ifKvDg3xaug6h", "mean_score": "0.39067220543806647", "stderr": "0.010317258861464411"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "qwen2-72b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4078282828282828, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2-72b-instruct", "source_row": {"Best score (across scorers)": "0.4078282828282828", "Country": "China", "Model version": "qwen2-72b-instruct", "Organization": "Alibaba", "Release date": "2024-06-07", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.02e+24", "id": "Pex6SpYJX8ZPySncfNNy7H", "mean_score": "0.4078282828282828", "stderr": "0.02574288051254005"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "llama-3-1-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.36678625377643503, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-70B-Instruct", "source_row": {"Best score (across scorers)": "0.36678625377643503", "Country": "United States of America", "Model version": "Llama-3.1-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.929e+24", "id": "6JiXiQ67iyiC7cFFRiogvs", "mean_score": "0.36678625377643503", "stderr": "0.010229604859690931"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "llama-3-1-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.03611111111111111, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-70B-Instruct", "source_row": {"Best score (across scorers)": "0.03611111111111111", "Country": "United States of America", "Model version": "Llama-3.1-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2025-02-25T14:05:03.270Z", "Training compute (FLOP)": "7.929e+24", "id": "X2tQXGELXtEbc2VX6gARd7", "mean_score": "0.03611111111111111", "stderr": "0.01691212350465157"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "llama-3-1-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.44191919191919193, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-70B-Instruct", "source_row": {"Best score (across scorers)": "0.44191919191919193", "Country": "United States of America", "Model version": "Llama-3.1-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.929e+24", "id": "MEwMcmSV32JHrsNoAq4BFL", "mean_score": "0.44191919191919193", "stderr": "0.024522863472149384"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "gemma-3-4b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.075, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-4b-it", "source_row": {"Best score (across scorers)": "0.075", "Country": "United States of America", "Model version": "gemma-3-4b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-28T08:35:58.000Z", "Training compute (FLOP)": "9.6e+22", "id": "KAPGpH2CLMafAt8MnAQJT3", "mean_score": "0.075", "stderr": "0.02763853991962833"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "gemma-3-4b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23232323232323232, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-4b-it", "source_row": {"Best score (across scorers)": "0.23232323232323232", "Country": "United States of America", "Model version": "gemma-3-4b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-28T08:35:26.000Z", "Training compute (FLOP)": "9.6e+22", "id": "McJyXLHBzx5VjAqXjmrrsb", "mean_score": "0.23232323232323232", "stderr": "0.018690142324718148"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-15", "model": "gemma-3-4b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.271044, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "google/gemma-3-4b-it", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "google/gemma-3-4b-it", "rows": [{"accuracy": "0.271044", "provider_name": "Parasail", "total_questions": "198"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gemma-3-4b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.1962, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Gemma-3-4b-it (Prompt)", "source_row": {"Model": "Gemma-3-4b-it (Prompt)", "Organization": "Google", "Overall Acc": "19.62%", "Rank": "101"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-13", "model": "llama-3-2-3b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.17340049999999999, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta-llama/llama-3.2-3b-instruct", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.2-3b-instruct", "rows": [{"accuracy": "0.17340049999999999", "provider_name": "Cloudflare", "total_questions": "396"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "llama-3-2-3b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.2195, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Llama-3.2-3B-Instruct (FC)", "source_row": {"Model": "Llama-3.2-3B-Instruct (FC)", "Organization": "Meta", "Overall Acc": "21.95%", "Rank": "98"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "wizardlm-2-8x22b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.25736404833836857, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "WizardLM-2-8x22B", "source_row": {"Best score (across scorers)": "0.25736404833836857", "Country": "United States of America", "Model version": "WizardLM-2-8x22B", "Organization": "Microsoft", "Release date": "2024-04-15", "Started at": "2025-01-27T00:00:00.000Z", "id": "4bAYQuh4heJoTNhapVZLNB", "mean_score": "0.25736404833836857", "stderr": "0.008719521575190186"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "wizardlm-2-8x22b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.43434343434343436, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "WizardLM-2-8x22B", "source_row": {"Best score (across scorers)": "0.43434343434343436", "Country": "United States of America", "Model version": "WizardLM-2-8x22B", "Organization": "Microsoft", "Release date": "2024-04-15", "Started at": "2025-01-27T00:00:00.000Z", "id": "MQJA3RUjsQzSubGJiukZ7A", "mean_score": "0.43434343434343436", "stderr": "0.023232093333444247"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gemini-1-5-flash-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2512273413897281, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-001", "source_row": {"Best score (across scorers)": "0.2512273413897281", "Country": "United States of America", "Model version": "gemini-1.5-flash-001", "Organization": "Google DeepMind", "Release date": "2024-05-23", "Started at": "2025-01-27T00:00:00.000Z", "id": "CPpd4hqQHtZ5pUejBELwKb", "mean_score": "0.2512273413897281", "stderr": "0.009456393246526343"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gemini-1-5-flash-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.03888888888888889, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-001", "source_row": {"Best score (across scorers)": "0.03888888888888889", "Country": "United States of America", "Model version": "gemini-1.5-flash-001", "Organization": "Google DeepMind", "Release date": "2024-05-23", "Started at": "2025-02-25T14:05:02.246Z", "id": "U9GVBhYjRzU55YVjna5mMT", "mean_score": "0.03888888888888889", "stderr": "0.014363726487520755"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gemini-1-5-flash-may-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.40372474747474746, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-001", "source_row": {"Best score (across scorers)": "0.40372474747474746", "Country": "United States of America", "Model version": "gemini-1.5-flash-001", "Organization": "Google DeepMind", "Release date": "2024-05-23", "Started at": "2025-01-27T00:00:00.000Z", "id": "Hc2ycMP6DyCZxCPjKbZFQU", "mean_score": "0.40372474747474746", "stderr": "0.023852475604024478"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gemma-2-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2788897280966767, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-2-27b-it", "source_row": {"Best score (across scorers)": "0.2788897280966767", "Country": "United States of America", "Model version": "gemma-2-27b-it", "Organization": "Google DeepMind", "Release date": "2024-06-24", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.106e+24", "id": "PM3GCQcVK7t2MujNp4F4kH", "mean_score": "0.2788897280966767", "stderr": "0.009939474306804167"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-07", "model": "gemma-2-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.013888888888888888, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-2-27b-it", "source_row": {"Best score (across scorers)": "0.013888888888888888", "Country": "United States of America", "Model version": "gemma-2-27b-it", "Organization": "Google DeepMind", "Release date": "2024-06-24", "Started at": "2025-03-07T23:11:28.104Z", "Training compute (FLOP)": "2.106e+24", "id": "jD6rJdvfNPErKEz9qkpssj", "mean_score": "0.013888888888888888", "stderr": "0.00907862525871088"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gemma-2-27b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3648989898989899, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-2-27b-it", "source_row": {"Best score (across scorers)": "0.3648989898989899", "Country": "United States of America", "Model version": "gemma-2-27b-it", "Organization": "Google DeepMind", "Release date": "2024-06-24", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.106e+24", "id": "ZUJxsJDpHnLXxNiDcvj22W", "mean_score": "0.3648989898989899", "stderr": "0.023897884428691632"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-28", "model": "gpt-4-jun-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.12, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-0613", "source_row": {"Best score (across scorers)": "0.12", "Country": "United States of America", "Model version": "gpt-4-0613", "Organization": "OpenAI", "Release date": "2023-06-13", "Started at": "2026-08-28T07:08:14.000Z", "Training compute (FLOP)": "2.1e+25", "id": "CvujZuqfj5r7qsa52Smyx9", "mean_score": "0.12", "stderr": "0.032659863237109045"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gpt-4-jun-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22970166163141995, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-0613", "source_row": {"Best score (across scorers)": "0.22970166163141995", "Country": "United States of America", "Model version": "gpt-4-0613", "Organization": "OpenAI", "Release date": "2023-06-13", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.1e+25", "id": "WsSrH8oHBm7m7vq9yTRqSC", "mean_score": "0.22970166163141995", "stderr": "0.008465534027077483"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-23", "model": "gpt-4-jun-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.011111111111111112, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-0613", "source_row": {"Best score (across scorers)": "0.011111111111111112", "Country": "United States of America", "Model version": "gpt-4-0613", "Organization": "OpenAI", "Release date": "2023-06-13", "Started at": "2025-10-23T16:09:26.069Z", "Training compute (FLOP)": "2.1e+25", "id": "hGVxg4EhSSJSCdH88m9soq", "mean_score": "0.011111111111111112", "stderr": "0.005362818328685681"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gpt-4-jun-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.30650252525252525, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-0613", "source_row": {"Best score (across scorers)": "0.30650252525252525", "Country": "United States of America", "Model version": "gpt-4-0613", "Organization": "OpenAI", "Release date": "2023-06-13", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.1e+25", "id": "fu7pWqbSC4MUBdwv3GFNa5", "mean_score": "0.30650252525252525", "stderr": "0.02337322948270623"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gemini-1-5-flash-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.04583333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-8b-001", "source_row": {"Best score (across scorers)": "0.04583333333333333", "Country": "United States of America", "Model version": "gemini-1.5-flash-8b-001", "Organization": "Google DeepMind", "Release date": "2024-10-03", "Started at": "2025-02-25T14:05:06.918Z", "id": "jGY5Ee5P9dyStA4Fw5Pa6q", "mean_score": "0.04583333333333333", "stderr": "0.020100756305184247"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-03-12", "model": "gemini-1-5-flash-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.32954545454545453, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.5-flash-8b-001", "source_row": {"Best score (across scorers)": "0.32954545454545453", "Country": "United States of America", "Model version": "gemini-1.5-flash-8b-001", "Organization": "Google DeepMind", "Release date": "2024-10-03", "Started at": "2025-03-12T21:23:34.734Z", "id": "gFT36W4KU3wggLQ9nNCctg", "mean_score": "0.32954545454545453", "stderr": "0.02416551116211757"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "llama-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.225547583081571, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Meta-Llama-3-70B-Instruct", "source_row": {"Best score (across scorers)": "0.225547583081571", "Country": "United States of America", "Model version": "Meta-Llama-3-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-04-18", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.861e+24", "id": "dekbZbU3BtiQZNKG8Txaax", "mean_score": "0.225547583081571", "stderr": "0.008257639939780867"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "llama-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.043055555555555555, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Meta-Llama-3-70B-Instruct", "source_row": {"Best score (across scorers)": "0.043055555555555555", "Country": "United States of America", "Model version": "Meta-Llama-3-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-04-18", "Started at": "2025-02-25T14:38:33.050Z", "Training compute (FLOP)": "7.861e+24", "id": "6RD3A6J432TZzxrKYRxph2", "mean_score": "0.043055555555555555", "stderr": "0.02192429710598427"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "llama-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.40561868686868685, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Meta-Llama-3-70B-Instruct", "source_row": {"Best score (across scorers)": "0.40561868686868685", "Country": "United States of America", "Model version": "Meta-Llama-3-70B-Instruct", "Organization": "Meta AI", "Release date": "2024-04-18", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.861e+24", "id": "jrvutCoFFG4KxYtU7REdpe", "mean_score": "0.40561868686868685", "stderr": "0.026224849492112068"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "mistral-large", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.24461858006042297, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2402", "source_row": {"Best score (across scorers)": "0.24461858006042297", "Country": "France", "Model version": "mistral-large-2402", "Organization": "Mistral AI", "Release date": "2024-02-26", "Started at": "2025-01-27T00:00:00.000Z", "id": "gypQecfFNK2x4pNGDtGMDv", "mean_score": "0.24461858006042297", "stderr": "0.00892334786643176"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "mistral-large", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.019444444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2402", "source_row": {"Best score (across scorers)": "0.019444444444444445", "Country": "France", "Model version": "mistral-large-2402", "Organization": "Mistral AI", "Release date": "2024-02-26", "Started at": "2025-02-25T14:05:07.542Z", "id": "ZVHvk3GTPUNTgjc5w3vkhW", "mean_score": "0.019444444444444445", "stderr": "0.00948945749721607"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "mistral-large", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.38762626262626265, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "mistral-large-2402", "source_row": {"Best score (across scorers)": "0.38762626262626265", "Country": "France", "Model version": "mistral-large-2402", "Organization": "Mistral AI", "Release date": "2024-02-26", "Started at": "2025-01-27T00:00:00.000Z", "id": "dDeREdDJ5bVyci3APZxzWb", "mean_score": "0.38762626262626265", "stderr": "0.024776598535417284"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "granite-4-0-1b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.008333333333333333, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "granite-4.0-1b", "source_row": {"Best score (across scorers)": "0.008333333333333333", "Model version": "granite-4.0-1b", "Release date": "2025-10-28", "Started at": "2026-08-30T22:46:30.000Z", "id": "PiFfhBoFbnaoqX5z2gVvra", "mean_score": "0.008333333333333333", "stderr": "0.004700633956814717"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "granite-4-0-1b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2398989898989899, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "granite-4.0-1b", "source_row": {"Best score (across scorers)": "0.2398989898989899", "Model version": "granite-4.0-1b", "Release date": "2025-10-28", "Started at": "2026-08-28T19:33:14.000Z", "id": "oWiE4CniV7k5adL5wDh5r8", "mean_score": "0.2398989898989899", "stderr": "0.01619451031298072"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "hermes-2-theta-llama-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22686933534743203, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Hermes-2-Theta-Llama-3-70B", "source_row": {"Best score (across scorers)": "0.22686933534743203", "Country": "United States of America", "Model version": "Hermes-2-Theta-Llama-3-70B", "Organization": "Nous Research,Arcee AI", "Release date": "2024-06-20", "Started at": "2025-01-27T00:00:00.000Z", "id": "ZAVTwsDqrvbWbqSgeNdXFd", "mean_score": "0.22686933534743203", "stderr": "0.008127708089237323"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-04-01", "model": "hermes-2-theta-llama-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Hermes-2-Theta-Llama-3-70B", "source_row": {"Best score (across scorers)": "0.025", "Country": "United States of America", "Model version": "Hermes-2-Theta-Llama-3-70B", "Organization": "Nous Research,Arcee AI", "Release date": "2024-06-20", "Started at": "2025-04-01T15:03:10.644Z", "id": "63MnE4e4AXvbQ585MaWTFv", "mean_score": "0.025", "stderr": "0.012309149097933271"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "hermes-2-theta-llama-3-70b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3746843434343434, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Hermes-2-Theta-Llama-3-70B", "source_row": {"Best score (across scorers)": "0.3746843434343434", "Country": "United States of America", "Model version": "Hermes-2-Theta-Llama-3-70B", "Organization": "Nous Research,Arcee AI", "Release date": "2024-06-20", "Started at": "2025-01-27T00:00:00.000Z", "id": "DHdeZNynu4VtRp2VGkCGvd", "mean_score": "0.3746843434343434", "stderr": "0.024803009993948306"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "mixtral-8x22b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.24244712990936557, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "open-mixtral-8x22b", "source_row": {"Best score (across scorers)": "0.24244712990936557", "Country": "France", "Model version": "open-mixtral-8x22b", "Organization": "Mistral AI", "Release date": "2024-04-17", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.34e+24", "id": "cGMb2vs9NhfkwZJsQXzLmQ", "mean_score": "0.24244712990936557", "stderr": "0.008296884112232158"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "mixtral-8x22b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.34059343434343436, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "open-mixtral-8x22b", "source_row": {"Best score (across scorers)": "0.34059343434343436", "Country": "France", "Model version": "open-mixtral-8x22b", "Organization": "Mistral AI", "Release date": "2024-04-17", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.34e+24", "id": "ifAM3DJBdm5eU9Y2DjaEvW", "mean_score": "0.34059343434343436", "stderr": "0.02068134779158647"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "llama-3-1-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.22875755287009064, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-8B-Instruct", "source_row": {"Best score (across scorers)": "0.22875755287009064", "Country": "United States of America", "Model version": "Llama-3.1-8B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "1.224e+24", "id": "gyxR3YTcmh4UcKZkET5pPX", "mean_score": "0.22875755287009064", "stderr": "0.008519457631995396"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-27", "model": "llama-3-1-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.016666666666666666, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-8B-Instruct", "source_row": {"Best score (across scorers)": "0.016666666666666666", "Country": "United States of America", "Model version": "Llama-3.1-8B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2026-08-27T16:29:11.000Z", "Training compute (FLOP)": "1.224e+24", "id": "6YAzFq89fwWiJ3uJdptCMr", "mean_score": "0.016666666666666666", "stderr": "0.009401267913629434"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-27", "model": "llama-3-1-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26957070707070707, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.1-8B-Instruct", "source_row": {"Best score (across scorers)": "0.26957070707070707", "Country": "United States of America", "Model version": "Llama-3.1-8B-Instruct", "Organization": "Meta AI", "Release date": "2024-07-23", "Started at": "2026-08-27T16:30:31.000Z", "Training compute (FLOP)": "1.224e+24", "id": "fxwRTtt8bLV4DNutwtMEox", "mean_score": "0.26957070707070707", "stderr": "0.018583813167150035"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-10-17", "model": "llama-3-1-8b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.26941151269035535, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "meta-llama/llama-3.1-8b-instruct", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "meta-llama/llama-3.1-8b-instruct", "rows": [{"accuracy": "0.22962980000000002", "provider_name": "Cloudflare", "total_questions": "990"}, {"accuracy": "0.27476285969556585", "provider_name": "CoreWeave", "total_questions": "1511"}, {"accuracy": "0.26941151269035535", "provider_name": "DeepInfra", "total_questions": "1773"}, {"accuracy": "0.280724", "provider_name": "Groq", "total_questions": "1584"}, {"accuracy": "0.25907233333333335", "provider_name": "Novita", "total_questions": "1782"}], "rule": "median over providers (5 of 5 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "mistral-7b-v0-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.002777777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Mistral-7B-Instruct-v0.3", "source_row": {"Best score (across scorers)": "0.002777777777777778", "Country": "France", "Model version": "Mistral-7B-Instruct-v0.3", "Organization": "Mistral AI", "Release date": "2024-05-27", "Started at": "2026-08-30T22:40:14.000Z", "id": "j6Mo25oV9Y9jdGxCMHZfpU", "mean_score": "0.002777777777777778", "stderr": "0.002777777777777778"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "mistral-7b-v0-3", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1518308080808081, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Mistral-7B-Instruct-v0.3", "source_row": {"Best score (across scorers)": "0.1518308080808081", "Country": "France", "Model version": "Mistral-7B-Instruct-v0.3", "Organization": "Mistral AI", "Release date": "2024-05-27", "Started at": "2025-01-27T00:00:00.000Z", "id": "YbQkMPC2GtNLhxqTEhhVsa", "mean_score": "0.1518308080808081", "stderr": "0.010148891037227303"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "granite-4-0-micro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.027777777777777776, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "granite-4.0-micro", "source_row": {"Best score (across scorers)": "0.027777777777777776", "Model version": "granite-4.0-micro", "Release date": "2025-10-02", "Started at": "2026-08-30T22:47:13.000Z", "id": "gVnaLSupMCiWmiScxfMLx3", "mean_score": "0.027777777777777776", "stderr": "0.014262647723679736"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "granite-4-0-micro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2828282828282828, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "granite-4.0-micro", "source_row": {"Best score (across scorers)": "0.2828282828282828", "Model version": "granite-4.0-micro", "Release date": "2025-10-02", "Started at": "2026-08-30T22:46:42.000Z", "id": "SadaSwnSdMpYRstc8zvgjJ", "mean_score": "0.2828282828282828", "stderr": "0.018237830822136196"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "claude-3-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.18174093655589124, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-sonnet-20240229", "source_row": {"Best score (across scorers)": "0.18174093655589124", "Country": "United States of America", "Model version": "claude-3-sonnet-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2025-01-27T00:00:00.000Z", "id": "FD2ztfjFUeqGyrtF8yuCUB", "mean_score": "0.18174093655589124", "stderr": "0.007981817344047291"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "claude-3-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-sonnet-20240229", "source_row": {"Best score (across scorers)": "0.025", "Country": "United States of America", "Model version": "claude-3-sonnet-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2025-02-25T14:05:05.872Z", "id": "AeRCEiG87bMFpxbxbmjCyn", "mean_score": "0.025", "stderr": "0.01950297827553939"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "claude-3-sonnet", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.4059343434343434, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-sonnet-20240229", "source_row": {"Best score (across scorers)": "0.4059343434343434", "Country": "United States of America", "Model version": "claude-3-sonnet-20240229", "Organization": "Anthropic", "Release date": "2024-02-29", "Started at": "2025-01-27T00:00:00.000Z", "id": "QLejPZXhjX6SQCQgzjNvCG", "mean_score": "0.4059343434343434", "stderr": "0.023920223491748827"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gemma-2-9b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2100641993957704, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-2-9b-it", "source_row": {"Best score (across scorers)": "0.2100641993957704", "Country": "United States of America", "Model version": "gemma-2-9b-it", "Organization": "Google DeepMind", "Release date": "2024-06-24", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "4.32e+23", "id": "Hdcqs42VnPNVBquXttexhq", "mean_score": "0.2100641993957704", "stderr": "0.008990724846256171"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-07", "model": "gemma-2-9b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.005555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-2-9b-it", "source_row": {"Best score (across scorers)": "0.005555555555555556", "Country": "United States of America", "Model version": "gemma-2-9b-it", "Organization": "Google DeepMind", "Release date": "2024-06-24", "Started at": "2025-03-07T23:11:27.610Z", "Training compute (FLOP)": "4.32e+23", "id": "WNE7cveVN8rsi7DmsFsFVy", "mean_score": "0.005555555555555556", "stderr": "0.003883473863441843"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gemma-2-9b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2746212121212121, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-2-9b-it", "source_row": {"Best score (across scorers)": "0.2746212121212121", "Country": "United States of America", "Model version": "gemma-2-9b-it", "Organization": "Google DeepMind", "Release date": "2024-06-24", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "4.32e+23", "id": "fQjoHKyUEPAuwdrBGfL8SG", "mean_score": "0.2746212121212121", "stderr": "0.023032467954778795"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "granite-4-0-350m", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.002777777777777778, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "granite-4.0-350m", "source_row": {"Best score (across scorers)": "0.002777777777777778", "Model version": "granite-4.0-350m", "Release date": "2025-10-28", "Started at": "2026-08-28T12:37:51.000Z", "id": "aQYiADNH3qVzZkWUmQn6nZ", "mean_score": "0.002777777777777778", "stderr": "0.002777777777777778"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "granite-4-0-350m", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.11237373737373738, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "granite-4.0-350m", "source_row": {"Best score (across scorers)": "0.11237373737373738", "Model version": "granite-4.0-350m", "Release date": "2025-10-28", "Started at": "2026-08-28T12:37:41.000Z", "id": "F6xZSehzKjpybEh8JkGwni", "mean_score": "0.11237373737373738", "stderr": "0.009083571075486867"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "granite-4-0-350m", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.1898, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Granite-4.0-350m (FC)", "source_row": {"Model": "Granite-4.0-350m (FC)", "Organization": "IBM", "Overall Acc": "18.98%", "Rank": "103"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-10-22", "model": "gpt-4-mar-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.005555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-0314", "source_row": {"Best score (across scorers)": "0.005555555555555556", "Country": "United States of America", "Model version": "gpt-4-0314", "Organization": "OpenAI", "Release date": "2023-03-14", "Started at": "2025-10-22T16:49:17.118Z", "Training compute (FLOP)": "2.1e+25", "id": "cLTgetxaaTCZoA9vwY4SdL", "mean_score": "0.005555555555555556", "stderr": "0.003883473863441843"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-10-23", "model": "gpt-4-mar-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3573232323232323, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-4-0314", "source_row": {"Best score (across scorers)": "0.3573232323232323", "Country": "United States of America", "Model version": "gpt-4-0314", "Organization": "OpenAI", "Release date": "2023-03-14", "Started at": "2025-10-23T12:16:00.615Z", "Training compute (FLOP)": "2.1e+25", "id": "FkbLkW5ed8rnZGFJMevDFn", "mean_score": "0.3573232323232323", "stderr": "0.02363099375599328"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-07", "model": "claude-2-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.019444444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-2.1", "source_row": {"Best score (across scorers)": "0.019444444444444445", "Country": "United States of America", "Model version": "claude-2.1", "Organization": "Anthropic", "Release date": "2023-11-21", "Started at": "2025-03-07T22:18:53.415Z", "id": "2vg9qw9pd8zSyRMu9YNiVm", "mean_score": "0.019444444444444445", "stderr": "0.010477515876628662"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "claude-2-1", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.32954545454545453, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-2.1", "source_row": {"Best score (across scorers)": "0.32954545454545453", "Country": "United States of America", "Model version": "claude-2.1", "Organization": "Anthropic", "Release date": "2023-11-21", "Started at": "2025-01-27T00:00:00.000Z", "id": "eMAisrJd7fAUHPfcPWXeXo", "mean_score": "0.32954545454545453", "stderr": "0.02324885083265289"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "eurus-2-7b-prime", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3390151515151515, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Eurus-2-7B-PRIME", "source_row": {"Best score (across scorers)": "0.3390151515151515", "Country": "China,United States of America", "Model version": "Eurus-2-7B-PRIME", "Organization": "Tsinghua University,University of Illinois Urbana-Champaign (UIUC),Shanghai AI Lab,Peking University,Shanghai Jiao Tong University,CUHK Shenzhen Research Institute", "Release date": "2024-12-31", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "4.01915904e+20", "id": "RJm4u2vXF6PUmThJGzAfHE", "mean_score": "0.3390151515151515", "stderr": "0.02107317439394748"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-31", "model": "phi-3-medium-14b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.17560422960725075, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Phi-3-medium-128k-instruct", "source_row": {"Best score (across scorers)": "0.17560422960725075", "Country": "United States of America", "Model version": "Phi-3-medium-128k-instruct", "Organization": "Microsoft", "Release date": "2024-04-23", "Started at": "2025-01-31T23:48:20.023Z", "Training compute (FLOP)": "4.032e+23", "id": "njA86GC9DzSXyiZWtUxXiD", "mean_score": "0.17560422960725075", "stderr": "0.007295657661098519"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-31", "model": "phi-3-medium-14b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2758838383838384, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Phi-3-medium-128k-instruct", "source_row": {"Best score (across scorers)": "0.2758838383838384", "Country": "United States of America", "Model version": "Phi-3-medium-128k-instruct", "Organization": "Microsoft", "Release date": "2024-04-23", "Started at": "2025-01-31T22:41:25.625Z", "Training compute (FLOP)": "4.032e+23", "id": "bDS2AGEf3AywbVcdDyAKZK", "mean_score": "0.2758838383838384", "stderr": "0.016117685500292727"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "qwen2-5-7b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-7b-instruct", "source_row": {"Best score (across scorers)": "0.025", "Country": "China", "Model version": "qwen2.5-7b-instruct", "Organization": "Alibaba", "Release date": "2024-09-19", "Started at": "2026-08-30T22:42:37.000Z", "Training compute (FLOP)": "8.2188e+23", "id": "U8HHNLLLVHJ7sHCfwdiDBX", "mean_score": "0.025", "stderr": "0.009401267913629434"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "qwen2-5-7b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3547979797979798, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "qwen2.5-7b-instruct", "source_row": {"Best score (across scorers)": "0.3547979797979798", "Country": "China", "Model version": "qwen2.5-7b-instruct", "Organization": "Alibaba", "Release date": "2024-09-19", "Started at": "2026-08-30T22:42:14.000Z", "Training compute (FLOP)": "8.2188e+23", "id": "6jkd5nJzspGZJtvZynZZTD", "mean_score": "0.3547979797979798", "stderr": "0.02291485990511179"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "or_gpqa_diamond", "eval_date": "2026-08-01", "model": "qwen2-5-7b", "retrieved_at": "2026-10-18T13:21:32Z", "score": 0.3417510000000001, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen-2.5-7b-instruct", "source_row": {"benchmark": "gpqa_diamond", "benchmark_page_url": "https://openrouter.ai/benchmarks/gpqa-diamond", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "rows": [{"accuracy": "0.36363599999999996", "provider_name": "Phala", "total_questions": "198"}, {"accuracy": "0.3417510000000001", "provider_name": "Together", "total_questions": "198"}], "rule": "median over providers (2 of 2 rows), lower median"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "or_tau2_airline", "eval_date": "2026-08-01", "model": "qwen2-5-7b", "retrieved_at": "2026-10-18T12:17:25Z", "score": 0.166667, "source": "OpenRouter benchmark datasets", "source_licence": "CC BY 4.0", "source_model_name": "qwen/qwen-2.5-7b-instruct", "source_row": {"benchmark": "tau_bench_verified_airline", "benchmark_page_url": "https://openrouter.ai/benchmarks/tau2-bench-airline", "model_permaslug": "qwen/qwen-2.5-7b-instruct", "rows": [{"accuracy": "0.166667", "provider_name": "Phala", "total_questions": "50"}], "rule": "median over providers (1 of 1 rows)"}, "source_sha256": "aa7b4c661b02ff153e48bf6cff39c6f09461b4b3bd56a680df9ef8a112128ee2", "source_snapshot": "https://openrouter.ai/api/v1/datasets/exports/benchmark-leaderboard/2026-10-18.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "claude-3-haiku", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1487915407854985, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-haiku-20240307", "source_row": {"Best score (across scorers)": "0.1487915407854985", "Country": "United States of America", "Model version": "claude-3-haiku-20240307", "Organization": "Anthropic", "Release date": "2024-03-07", "Started at": "2025-01-27T00:00:00.000Z", "id": "QSnWVKUVj9six4jYj4HJD9", "mean_score": "0.1487915407854985", "stderr": "0.007229965913719499"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "claude-3-haiku", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.018055555555555554, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-haiku-20240307", "source_row": {"Best score (across scorers)": "0.018055555555555554", "Country": "United States of America", "Model version": "claude-3-haiku-20240307", "Organization": "Anthropic", "Release date": "2024-03-07", "Started at": "2025-02-25T14:05:02.017Z", "id": "ThMd6DNDTtyWxhPcb9Kq4y", "mean_score": "0.018055555555555554", "stderr": "0.008098544298396251"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "claude-3-haiku", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3630050505050505, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-3-haiku-20240307", "source_row": {"Best score (across scorers)": "0.3630050505050505", "Country": "United States of America", "Model version": "claude-3-haiku-20240307", "Organization": "Anthropic", "Release date": "2024-03-07", "Started at": "2025-01-27T00:00:00.000Z", "id": "T3FWdCCWBryVjxwTcoY5bK", "mean_score": "0.3630050505050505", "stderr": "0.024180910672074975"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gpt-3-5-turbo-nov-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.15889350453172205, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-3.5-turbo-1106", "source_row": {"Best score (across scorers)": "0.15889350453172205", "Country": "United States of America", "Model version": "gpt-3.5-turbo-1106", "Organization": "OpenAI", "Release date": "2023-11-06", "Started at": "2025-01-27T00:00:00.000Z", "id": "cJWh887zSiTtnxm9XAJVkr", "mean_score": "0.15889350453172205", "stderr": "0.006959408516001194"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gpt-3-5-turbo-nov-2023", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2803030303030303, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-3.5-turbo-1106", "source_row": {"Best score (across scorers)": "0.2803030303030303", "Country": "United States of America", "Model version": "gpt-3.5-turbo-1106", "Organization": "OpenAI", "Release date": "2023-11-06", "Started at": "2025-01-27T00:00:00.000Z", "id": "enhRcQmfN6RFB2zu7R3Uxr", "mean_score": "0.2803030303030303", "stderr": "0.018720372561004013"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "ministral-3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14444864048338368, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "ministral-3b-2410", "source_row": {"Best score (across scorers)": "0.14444864048338368", "Country": "France", "Model version": "ministral-3b-2410", "Organization": "Mistral AI", "Release date": "2024-10-16", "Started at": "2025-01-27T00:00:00.000Z", "id": "YNhHxRT9Ee2uEWWJh9RXuP", "mean_score": "0.14444864048338368", "stderr": "0.007446287617112978"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "ministral-3b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.25252525252525254, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "ministral-3b-2410", "source_row": {"Best score (across scorers)": "0.25252525252525254", "Country": "France", "Model version": "ministral-3b-2410", "Organization": "Mistral AI", "Release date": "2024-10-16", "Started at": "2025-01-27T00:00:00.000Z", "id": "bJV4D8SEo9zMJPEYTTXJNZ", "mean_score": "0.25252525252525254", "stderr": "0.019305614722797293"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "claude-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1172583081570997, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-2.0", "source_row": {"Best score (across scorers)": "0.1172583081570997", "Country": "United States of America", "Model version": "claude-2.0", "Organization": "Anthropic", "Release date": "2023-07-11", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.866e+24", "id": "Gs4vxvDD5wmrSvyTkqx2YV", "mean_score": "0.1172583081570997", "stderr": "0.006595670913969979"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-03-07", "model": "claude-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-2.0", "source_row": {"Best score (across scorers)": "0.025", "Country": "United States of America", "Model version": "claude-2.0", "Organization": "Anthropic", "Release date": "2023-07-11", "Started at": "2025-03-07T22:18:52.824Z", "Training compute (FLOP)": "3.866e+24", "id": "7LLTmqs4LC5GLVfJEqJWSC", "mean_score": "0.025", "stderr": "0.015179897658722631"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "claude-2", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3465909090909091, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "claude-2.0", "source_row": {"Best score (across scorers)": "0.3465909090909091", "Country": "United States of America", "Model version": "claude-2.0", "Organization": "Anthropic", "Release date": "2023-07-11", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "3.866e+24", "id": "mTa4f52zsEuxdrPy7WfQvX", "mean_score": "0.3465909090909091", "stderr": "0.025076255983551475"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "mystery_game_puzzles", "eval_date": "2026-08-27", "model": "gpt-3-5-turbo-jan-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.03, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-3.5-turbo-0125", "source_row": {"Best score (across scorers)": "0.03", "Country": "United States of America", "Model version": "gpt-3.5-turbo-0125", "Organization": "OpenAI", "Release date": "2024-01-25", "Started at": "2026-08-27T17:57:53.000Z", "id": "UfmijCsx59sZ8xpcbcWDms", "mean_score": "0.03", "stderr": "0.01714466079977653"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#mystery_game_puzzles.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "frontiermath_t13", "eval_date": "2026-08-27", "model": "gpt-3-5-turbo-jan-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-3.5-turbo-0125", "source_row": {"Best score (across scorers)": "0.0", "Country": "United States of America", "Model version": "gpt-3.5-turbo-0125", "Organization": "OpenAI", "Release date": "2024-01-25", "Started at": "2026-08-27T19:59:25.000Z", "id": "FNaBAvmeTiRRFDW63KemZ7", "mean_score": "0.0", "stderr": "0.0"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#frontiermath_tiers_1_3_v2.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gpt-3-5-turbo-jan-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1163141993957704, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-3.5-turbo-0125", "source_row": {"Best score (across scorers)": "0.1163141993957704", "Country": "United States of America", "Model version": "gpt-3.5-turbo-0125", "Organization": "OpenAI", "Release date": "2024-01-25", "Started at": "2025-01-27T00:00:00.000Z", "id": "9tSgxGV3nzhTBpq7GcX7yx", "mean_score": "0.1163141993957704", "stderr": "0.005543945913851348"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-07", "model": "gpt-3-5-turbo-jan-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.022222222222222223, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-3.5-turbo-0125", "source_row": {"Best score (across scorers)": "0.022222222222222223", "Country": "United States of America", "Model version": "gpt-3.5-turbo-0125", "Organization": "OpenAI", "Release date": "2024-01-25", "Started at": "2026-08-07T00:00:50.000Z", "id": "7ZVhkukwaRXU2Sx2ttibPF", "mean_score": "0.022222222222222223", "stderr": "0.022222222222222223"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gpt-3-5-turbo-jan-2024", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.27178030303030304, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gpt-3.5-turbo-0125", "source_row": {"Best score (across scorers)": "0.27178030303030304", "Country": "United States of America", "Model version": "gpt-3.5-turbo-0125", "Organization": "OpenAI", "Release date": "2024-01-25", "Started at": "2025-01-27T00:00:00.000Z", "id": "eZaQhpbhbzbo3sbKGS5Efa", "mean_score": "0.27178030303030304", "stderr": "0.01953579968788737"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "ministral-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.14935800604229607, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "ministral-8b-2410", "source_row": {"Best score (across scorers)": "0.14935800604229607", "Country": "France", "Model version": "ministral-8b-2410", "Organization": "Mistral AI", "Release date": "2024-10-16", "Started at": "2025-01-27T00:00:00.000Z", "id": "hqdiKcvEPSRyvenUVB9qxM", "mean_score": "0.14935800604229607", "stderr": "0.007454534670584494"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "ministral-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.27146464646464646, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "ministral-8b-2410", "source_row": {"Best score (across scorers)": "0.27146464646464646", "Country": "France", "Model version": "ministral-8b-2410", "Organization": "Mistral AI", "Release date": "2024-10-16", "Started at": "2025-01-27T00:00:00.000Z", "id": "DGEzZA6nvTne4MEGSuo3Tm", "mean_score": "0.27146464646464646", "stderr": "0.020863706671553732"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "ministral-8b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.111, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Ministral-8B-Instruct-2410 (FC)", "source_row": {"Model": "Ministral-8B-Instruct-2410 (FC)", "Organization": "Mistral AI", "Overall Acc": "11.10%", "Rank": "105"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "mistral-nemo", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1082892749244713, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "open-mistral-nemo-2407", "source_row": {"Best score (across scorers)": "0.1082892749244713", "Country": "France", "Model version": "open-mistral-nemo-2407", "Organization": "Mistral AI", "Release date": "2024-07-18", "Started at": "2025-01-27T00:00:00.000Z", "id": "9qCrggpoq8FjQWVQhHaGBT", "mean_score": "0.1082892749244713", "stderr": "0.006092598712054613"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "mistral-nemo", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.2989267676767677, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "open-mistral-nemo-2407", "source_row": {"Best score (across scorers)": "0.2989267676767677", "Country": "France", "Model version": "open-mistral-nemo-2407", "Organization": "Mistral AI", "Release date": "2024-07-18", "Started at": "2025-01-27T00:00:00.000Z", "id": "XCSWtbDquA7FUiWfd7kh9q", "mean_score": "0.2989267676767677", "stderr": "0.02096191098761666"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "mistral-nemo", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.2763, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Open-Mistral-Nemo-2407 (FC)", "source_row": {"Model": "Open-Mistral-Nemo-2407 (FC)", "Organization": "Mistral AI", "Overall Acc": "27.63%", "Rank": "78"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "gemini-1-0-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.11244335347432025, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.0-pro-001", "source_row": {"Best score (across scorers)": "0.11244335347432025", "Country": "United States of America", "Model version": "gemini-1.0-pro-001", "Organization": "Google DeepMind", "Release date": "2023-12-13", "Started at": "2025-01-27T00:00:00.000Z", "id": "iUdecHq9vU5utytzgTXzqY", "mean_score": "0.11244335347432025", "stderr": "0.005990949270731778"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2025-02-25", "model": "gemini-1-0-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.011111111111111112, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.0-pro-001", "source_row": {"Best score (across scorers)": "0.011111111111111112", "Country": "United States of America", "Model version": "gemini-1.0-pro-001", "Organization": "Google DeepMind", "Release date": "2023-12-13", "Started at": "2025-02-25T14:05:05.526Z", "id": "NpJvYQDcmwbGgS5MzZUPUg", "mean_score": "0.011111111111111112", "stderr": "0.006054026310473157"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "gemini-1-0-pro", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.33964646464646464, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemini-1.0-pro-001", "source_row": {"Best score (across scorers)": "0.33964646464646464", "Country": "United States of America", "Model version": "gemini-1.0-pro-001", "Organization": "Google DeepMind", "Release date": "2023-12-13", "Started at": "2025-01-27T00:00:00.000Z", "id": "ibpS8bs4z3wD3Hb6F3kpdV", "mean_score": "0.33964646464646464", "stderr": "0.020110933405176737"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "dbrx", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.11650302114803625, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "dbrx-instruct", "source_row": {"Best score (across scorers)": "0.11650302114803625", "Country": "United States of America", "Model version": "dbrx-instruct", "Organization": "Databricks", "Release date": "2024-03-27", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.6e+24", "id": "PMbefrjykQmTBpABMa7FJi", "mean_score": "0.11650302114803625", "stderr": "0.007355853708151387"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "dbrx", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.32891414141414144, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "dbrx-instruct", "source_row": {"Best score (across scorers)": "0.32891414141414144", "Country": "United States of America", "Model version": "dbrx-instruct", "Organization": "Databricks", "Release date": "2024-03-27", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "2.6e+24", "id": "TTa3hu7PgRxHgizVreZsdJ", "mean_score": "0.32891414141414144", "stderr": "0.03256944333564935"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "mixtral-8x7b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.09950906344410876, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "open-mixtral-8x7b", "source_row": {"Best score (across scorers)": "0.09950906344410876", "Country": "France", "Model version": "open-mixtral-8x7b", "Organization": "Mistral AI", "Release date": "2023-12-11", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.74e+23", "id": "GN3Hez798byishFw3U6cpp", "mean_score": "0.09950906344410876", "stderr": "0.005303272245515919"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "mixtral-8x7b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.3058712121212121, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Mixtral-8x7B-Instruct-v0.1", "source_row": {"Best score (across scorers)": "0.3058712121212121", "Country": "France", "Model version": "Mixtral-8x7B-Instruct-v0.1", "Organization": "Mistral AI", "Release date": "2023-12-11", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.74e+23", "id": "Sn2qbB2HiySe7NcG28jnyM", "mean_score": "0.3058712121212121", "stderr": "0.01937059603941351"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "llama-3-2-1b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.005555555555555556, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.2-1B-Instruct", "source_row": {"Best score (across scorers)": "0.005555555555555556", "Country": "United States of America", "Model version": "Llama-3.2-1B-Instruct", "Organization": "Meta AI", "Release date": "2024-09-24", "Started at": "2026-08-30T22:39:12.000Z", "Training compute (FLOP)": "6.642e+22", "id": "8qwT2CF9U5zV99PFirnVsp", "mean_score": "0.005555555555555556", "stderr": "0.003883473863441843"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-30", "model": "llama-3-2-1b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.23926767676767677, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Llama-3.2-1B-Instruct", "source_row": {"Best score (across scorers)": "0.23926767676767677", "Country": "United States of America", "Model version": "Llama-3.2-1B-Instruct", "Organization": "Meta AI", "Release date": "2024-09-24", "Started at": "2026-08-30T22:39:11.000Z", "Training compute (FLOP)": "6.642e+22", "id": "FGEsLTxzZ7fE9aq7DhdqZb", "mean_score": "0.23926767676767677", "stderr": "0.016278803912773237"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "llama-3-2-1b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.1082, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Llama-3.2-1B-Instruct (FC)", "source_row": {"Model": "Llama-3.2-1B-Instruct (FC)", "Organization": "Meta", "Overall Acc": "10.82%", "Rank": "107"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}
{"benchmark": "math_level_5", "eval_date": "2025-01-27", "model": "llama-3-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.0612726586102719, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Meta-Llama-3-8B-Instruct", "source_row": {"Best score (across scorers)": "0.0612726586102719", "Country": "United States of America", "Model version": "Meta-Llama-3-8B-Instruct", "Organization": "Meta AI", "Release date": "2024-04-18", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.2e+23", "id": "BURdKU7mhUX4YMPUDoFsed", "mean_score": "0.0612726586102719", "stderr": "0.0037433164863856527"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#math_level_5.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-30", "model": "llama-3-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.019444444444444445, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Meta-Llama-3-8B-Instruct", "source_row": {"Best score (across scorers)": "0.019444444444444445", "Country": "United States of America", "Model version": "Meta-Llama-3-8B-Instruct", "Organization": "Meta AI", "Release date": "2024-04-18", "Started at": "2026-08-30T22:38:53.000Z", "Training compute (FLOP)": "7.2e+23", "id": "CAcNPzYBEu39HFWPmuUEeP", "mean_score": "0.019444444444444445", "stderr": "0.010477515876628662"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2025-01-27", "model": "llama-3-8b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.26073232323232326, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "Meta-Llama-3-8B-Instruct", "source_row": {"Best score (across scorers)": "0.26073232323232326", "Country": "United States of America", "Model version": "Meta-Llama-3-8B-Instruct", "Organization": "Meta AI", "Release date": "2024-04-18", "Started at": "2025-01-27T00:00:00.000Z", "Training compute (FLOP)": "7.2e+23", "id": "iT3BBFWjsZk2rKHKeJDsM8", "mean_score": "0.26073232323232326", "stderr": "0.01703043361773576"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "otis_mock_aime", "eval_date": "2026-08-28", "model": "gemma-3-1b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.011111111111111112, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-1b-it", "source_row": {"Best score (across scorers)": "0.011111111111111112", "Country": "United States of America", "Model version": "gemma-3-1b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-28T08:28:18.000Z", "Training compute (FLOP)": "1.2e+22", "id": "3CGChL5vNDM6Df5C8Z8owN", "mean_score": "0.011111111111111112", "stderr": "0.005362818328685681"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#otis_mock_aime_2024_2025.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "gpqa_diamond", "eval_date": "2026-08-28", "model": "gemma-3-1b", "retrieved_at": "2026-10-18T12:17:21Z", "score": 0.1994949494949495, "source": "Epoch AI Benchmarking Hub", "source_licence": "CC BY 4.0", "source_model_name": "gemma-3-1b-it", "source_row": {"Best score (across scorers)": "0.1994949494949495", "Country": "United States of America", "Model version": "gemma-3-1b-it", "Organization": "Google DeepMind", "Release date": "2025-03-12", "Started at": "2026-08-28T08:27:48.000Z", "Training compute (FLOP)": "1.2e+22", "id": "Pc6CyTEYKbcdDUDKzGZnS9", "mean_score": "0.1994949494949495", "stderr": "0.01521201537793133"}, "source_sha256": "e8797a71c56eb5dcb04037f3c03711e0a144619adde407df62d65bb2acf2fee6", "source_snapshot": "https://epoch.ai/data/benchmark_data.zip#gpqa_diamond.csv@sha256:e8797a71c56eb5dc"}
{"benchmark": "bfcl_overall", "eval_date": "", "model": "gemma-3-1b", "retrieved_at": "2026-10-18T12:17:30Z", "score": 0.0717, "source": "Berkeley Function Calling Leaderboard", "source_licence": "Apache-2.0", "source_model_name": "Gemma-3-1b-it (Prompt)", "source_row": {"Model": "Gemma-3-1b-it (Prompt)", "Organization": "Google", "Overall Acc": "7.17%", "Rank": "109"}, "source_sha256": "ede5081a24bc5ac7f9b3b09a52af6bcfcd35aa0f1709239fed979db77a8159a6", "source_snapshot": "https://raw.githubusercontent.com/ShishirPatil/gorilla/927b29cc0f96061898a3e568ceccbc8bc81ee6a4/data_overall.csv"}