{
 "publisher": "AI vs Games",
 "version": "2026.10.18",
 "generated_at": "2026-10-18T16:32:28-04:00",
 "license": "CC BY 4.0",
 "attribution": "AI vs Games (aivsgames.com) and the original sources linked in each item",
 "rule": "Results are per game. Do not combine them into cross-game rankings.",
 "runs": [
  {
   "id": "aoe2-llm-arena",
   "kind": "run",
   "game": "age-of-empires-2",
   "game_name": "Age of Empires II",
   "title": "AoE2 LLM Arena",
   "status": "ARCHIVE",
   "summary": "Claude, GPT, Gemini and Kimi K2 wrote AoE2 AI scripts and played each other. A video was published Aug 18, 2026, and Claude reportedly performed best.",
   "fetcher": null,
   "results": [
    {
     "model": "Claude",
     "metric": "Result",
     "value": "Reportedly best (military & relic wins)",
     "as_of": "2026-08-18",
     "source_url": "https://www.martondobos.com/posts/aoe2-llm-arena-1-why-teaching-claude",
     "verification": "self-reported"
    }
   ],
   "sources": [
    "https://www.martondobos.com/posts/aoe2-llm-arena-1-why-teaching-claude",
    "https://daily.dev/posts/ai-plays-age-of-empires-ii-cbfqxg1w5"
   ],
   "flags": [],
   "last_checked": "2026-10-17",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "manual source"
   },
   "section": "more",
   "config": {
    "harness": "not disclosed",
    "inputs": "not disclosed",
    "tools": "not disclosed",
    "assistance": "not disclosed",
    "budget": "not disclosed"
   },
   "url": "https://aivsgames.com/games/age-of-empires-2/#aoe2-llm-arena"
  },
  {
   "id": "balatrobench",
   "kind": "leaderboard",
   "game": "balatro",
   "game_name": "Balatro",
   "title": "BalatroBench",
   "status": "ARCHIVE",
   "summary": "Ranks LLMs by the average round they reach in Balatro when playing through tool calls. The live table loads client-side, so the current standings are not verified.",
   "fetcher": "balatrobench",
   "results": [
    {
     "model": "gemini-3-pro-preview",
     "metric": "Avg final round",
     "value": "19.3 ± 6.8",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "9/15 runs won",
     "rank": 1
    },
    {
     "model": "gpt-5.2",
     "metric": "Avg final round",
     "value": "18.4 ± 5.1",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "3/16 runs won",
     "rank": 2
    },
    {
     "model": "gemini-3-flash-preview",
     "metric": "Avg final round",
     "value": "16.7 ± 7.4",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "5/15 runs won",
     "rank": 3
    },
    {
     "model": "claude-opus-4.5",
     "metric": "Avg final round",
     "value": "13.9 ± 6.5",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "2/15 runs won",
     "rank": 4
    },
    {
     "model": "claude-sonnet-4.5",
     "metric": "Avg final round",
     "value": "11.9 ± 6.7",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "2/15 runs won",
     "rank": 5
    },
    {
     "model": "grok-4",
     "metric": "Avg final round",
     "value": "9.5 ± 4.9",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "0/15 runs won",
     "rank": 6
    },
    {
     "model": "deepseek-v3.2",
     "metric": "Avg final round",
     "value": "8.7 ± 6.8",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "0/15 runs won",
     "rank": 7
    },
    {
     "model": "grok-4.1-fast",
     "metric": "Avg final round",
     "value": "7.7 ± 5.5",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "0/15 runs won",
     "rank": 8
    },
    {
     "model": "qwen3-max",
     "metric": "Avg final round",
     "value": "7.3 ± 4.3",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "0/15 runs won",
     "rank": 9
    },
    {
     "model": "gpt-oss-120b",
     "metric": "Avg final round",
     "value": "6.3 ± 2.9",
     "as_of": "2026-02-03",
     "source_url": "https://balatrobench.com/",
     "verification": "verified",
     "note": "0/15 runs won",
     "rank": 10
    }
   ],
   "flags": [],
   "sources": [
    "https://balatrobench.com/",
    "https://github.com/coder/balatrobench"
   ],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "auto_flags": [
    "Leaderboard data generated 2026-02-03 (benchmark version v1.0.8)."
   ],
   "section": "more",
   "config": {
    "harness": "BalatroBench default strategy",
    "inputs": "game state (text)",
    "tools": "not disclosed",
    "assistance": "none",
    "budget": "not disclosed"
   },
   "url": "https://aivsgames.com/games/balatro/#balatrobench"
  },
  {
   "id": "balrog",
   "kind": "leaderboard",
   "game": "nethack",
   "game_name": "NetHack & roguelikes (BALROG)",
   "title": "BALROG: BabyAI, Crafter, TextWorld, BabaIsAI, MiniHack, NetHack",
   "status": "RECENT",
   "summary": "Average progress across 6 game environments. NetHack is still the hardest. The leaderboard updates weekly on Mondays.",
   "fetcher": "balrog",
   "results": [
    {
     "model": "GPT-6-Astra-Max",
     "metric": "Avg progress %",
     "value": 68.3,
     "as_of": "2026-09-18",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 13.2%",
     "rank": 1
    },
    {
     "model": "Claude-Opus-5-Max",
     "metric": "Avg progress %",
     "value": 63.4,
     "as_of": "2026-09-20",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 7.3%",
     "rank": 2
    },
    {
     "model": "GPT-5.6-Sol-Max",
     "metric": "Avg progress %",
     "value": 60.0,
     "as_of": "2026-09-18",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 3.2%",
     "rank": 3
    },
    {
     "model": "Gemini-3-Pro",
     "metric": "Avg progress %",
     "value": 58.1,
     "as_of": "2026-02-03",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 6.8%",
     "rank": 4
    },
    {
     "model": "Gemini-3.1-Pro-Thinking",
     "metric": "Avg progress %",
     "value": 57.0,
     "as_of": "2026-02-25",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 2.6%",
     "rank": 5
    },
    {
     "model": "Gemini-3.1-Pro",
     "metric": "Avg progress %",
     "value": 56.9,
     "as_of": "2026-02-21",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 3.0%",
     "rank": 6
    },
    {
     "model": "GPT-5.6-Terra-Max",
     "metric": "Avg progress %",
     "value": 53.2,
     "as_of": "2026-09-20",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 3.3%",
     "rank": 7
    },
    {
     "model": "Gemini-3-Flash",
     "metric": "Avg progress %",
     "value": 48.1,
     "as_of": "2026-02-13",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 4.0%",
     "rank": 8
    },
    {
     "model": "GPT-5.6-Luna-Max",
     "metric": "Avg progress %",
     "value": 45.6,
     "as_of": "2026-09-18",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 1.7%",
     "rank": 9
    },
    {
     "model": "Grok-4",
     "metric": "Avg progress %",
     "value": 43.6,
     "as_of": "2025-07-23",
     "source_url": "https://balrogai.com/",
     "verification": "verified",
     "note": "NetHack 1.8%",
     "rank": 10
    }
   ],
   "sources": [
    "https://balrogai.com/"
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "section": "more",
   "config": {
    "harness": "BALROG agentic harness",
    "inputs": "text / vision per track",
    "tools": "none",
    "assistance": "none",
    "budget": "not disclosed"
   },
   "url": "https://aivsgames.com/games/nethack/#balrog"
  },
  {
   "id": "claude-plays-pokemon-red",
   "kind": "run",
   "game": "pokemon-red",
   "game_name": "Pokémon Red",
   "title": "ClaudePlaysPokemon: Red (Claude Opus 4.7)",
   "status": "PAUSED",
   "summary": "Claude Opus 4.7's Pokémon Red run came back online on May 14, 2026. The channel has not streamed since mid-May.",
   "fetcher": "twitch_status",
   "twitch_login": "claudeplayspokemon",
   "results": [
    {
     "model": "Claude Opus 4.7",
     "metric": "Stream status",
     "value": "No streams Sep 17 – Oct 17, 2026",
     "as_of": "2026-10-17",
     "source_url": "https://www.twitchmetrics.net/c/1165258274-claudeplayspokemon",
     "verification": "verified",
     "note": "Last seen May 18, 2026"
    },
    {
     "model": "Claude Opus 4.7",
     "metric": "Twitch followers",
     "value": "15.2K",
     "as_of": "2026-10-17",
     "source_url": "https://www.twitch.tv/claudeplayspokemon",
     "verification": "verified"
    },
    {
     "model": "Claude Opus 4.7",
     "metric": "Live now",
     "value": "no",
     "as_of": "2026-10-18",
     "source_url": "https://www.twitch.tv/claudeplayspokemon",
     "verification": "verified",
     "note": "Checked via a public uptime endpoint (decapi.me). Move to the official Twitch API once a dev app exists."
    }
   ],
   "sources": [
    "https://www.twitch.tv/claudeplayspokemon",
    "https://www.twitchmetrics.net/c/1165258274-claudeplayspokemon",
    "https://www.reddit.com/r/ClaudePlaysPokemon/",
    "https://x.com/AnthropicAI/status/1894419042150027701"
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "Twitch: waiting for an official Twitch API app. No scraping. Last known status kept with its date."
   },
   "section": "pokemon",
   "config": {
    "harness": "ClaudePlaysPokemon harness",
    "inputs": "not disclosed",
    "tools": "not disclosed",
    "assistance": "not disclosed",
    "budget": "not disclosed"
   },
   "creator": {
    "name": "ClaudePlaysPokemon",
    "links": [
     {
      "label": "Twitch channel",
      "url": "https://www.twitch.tv/claudeplayspokemon"
     },
     {
      "label": "Community subreddit",
      "url": "https://www.reddit.com/r/ClaudePlaysPokemon/"
     }
    ]
   },
   "project_started": {
    "date": "2025-02-25",
    "label": "Anthropic on X (Feb 25, 2025): 'Claude Plays Pokémon continues on as a researcher's personal project. Follow along on Twitch'",
    "url": "https://x.com/AnthropicAI/status/1894419042150027701",
    "also": [
     {
      "label": "Anthropic: Claude's extended thinking (Feb 24, 2025), section 'Claude plays Pokémon'",
      "url": "https://www.anthropic.com/research/visible-extended-thinking"
     },
     {
      "label": "TechCrunch (Feb 25, 2025): Anthropic launched Claude Plays Pokémon on Twitch on Tuesday afternoon",
      "url": "https://techcrunch.com/2025/02/25/anthropics-claude-ai-is-playing-pokemon-on-twitch-slowly/"
     }
    ],
    "note": "Public Twitch stream started Feb 25, 2025 with Claude 3.7 Sonnet playing Pokémon Red. Anthropic says researchers had tested earlier Claude models on Pokémon privately over the year before."
   },
   "url": "https://aivsgames.com/games/pokemon-red/#claude-plays-pokemon-red"
  },
  {
   "id": "factorio-fle",
   "kind": "leaderboard",
   "game": "factorio",
   "game_name": "Factorio",
   "title": "Factorio Learning Environment",
   "status": "ARCHIVE",
   "summary": "Agents write programs to build factories and are scored by production. The data is stale: only older models are listed, and the official page currently shows zeros.",
   "fetcher": "benchmarklist_fle",
   "results": [
    {
     "model": "Claude 3.5-Sonnet",
     "metric": "Production score",
     "value": 293206,
     "as_of": "2026-05-28",
     "source_url": "https://benchmarklist.com/benchmarks/factorio_learning_environment/",
     "verification": "verified",
     "rank": 1
    },
    {
     "model": "Gemini-2-Flash",
     "metric": "Production score",
     "value": 115782,
     "as_of": "2026-05-28",
     "source_url": "https://benchmarklist.com/benchmarks/factorio_learning_environment/",
     "verification": "verified",
     "rank": 2
    },
    {
     "model": "GPT4o",
     "metric": "Production score",
     "value": 87599,
     "as_of": "2026-05-28",
     "source_url": "https://benchmarklist.com/benchmarks/factorio_learning_environment/",
     "verification": "verified",
     "rank": 3
    },
    {
     "model": "Llama-3.3-70b",
     "metric": "Production score",
     "value": 54998,
     "as_of": "2026-05-28",
     "source_url": "https://benchmarklist.com/benchmarks/factorio_learning_environment/",
     "verification": "verified",
     "rank": 4
    },
    {
     "model": "Deepseek-v3",
     "metric": "Production score",
     "value": 48585,
     "as_of": "2026-05-28",
     "source_url": "https://benchmarklist.com/benchmarks/factorio_learning_environment/",
     "verification": "verified",
     "rank": 5
    },
    {
     "model": "GPT4o-Mini",
     "metric": "Production score",
     "value": 26756,
     "as_of": "2026-05-28",
     "source_url": "https://benchmarklist.com/benchmarks/factorio_learning_environment/",
     "verification": "verified",
     "rank": 6
    }
   ],
   "flags": [
    "Stale: no current-generation models listed."
   ],
   "sources": [
    "https://jackhopkins.github.io/factorio-learning-environment/leaderboard/",
    "https://benchmarklist.com/benchmarks/factorio_learning_environment/"
   ],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": false,
    "checked": "2026-10-18T16:31:30-04:00",
    "error": "403 Client Error: Forbidden for url: https://benchmarklist.com/benchmarks/factorio_learning_environment/",
    "note": "kept last good data"
   },
   "section": "more",
   "config": {
    "harness": "Factorio Learning Environment (code API)",
    "inputs": "text / code",
    "tools": "FLE Python API",
    "assistance": "none",
    "budget": "not disclosed"
   },
   "url": "https://aivsgames.com/games/factorio/#factorio-fle"
  },
  {
   "id": "gemini-flash-race-pokemon-blue",
   "kind": "run",
   "game": "pokemon-blue",
   "game_name": "Pokémon Blue",
   "title": "Gemini Flash 'vision-first' series: Pokémon Blue",
   "status": "RECENT",
   "summary": "A series of 'vision-first' Pokémon Blue runs with successive Gemini Flash models on the same harness. Gemini 3.8 Flash: 8 badges; Gemini 3.7 Flash: 8 badges; Gemini 3.5 Flash: 6 badges. Each run's notepad on the public repo names the game as Pokémon Blue.",
   "fetcher": "gpp_race",
   "results": [
    {
     "model": "Gemini 3.8 Flash",
     "metric": "Badges",
     "value": 8,
     "as_of": "2026-09-27",
     "source_url": "https://gpp-viewer.arisef.org/",
     "verification": "verified",
     "note": "Hub run vision_first_gemini_3_8_flash: idle, 47,889 turns",
     "rank": 1
    },
    {
     "model": "Gemini 3.7 Flash",
     "metric": "Badges",
     "value": 8,
     "as_of": "2026-09-19",
     "source_url": "https://gpp-viewer.arisef.org/",
     "verification": "verified",
     "note": "Hub run vision_first_gemini_3_7_flash: idle, 55,601 turns",
     "rank": 2
    },
    {
     "model": "Gemini 3.5 Flash",
     "metric": "Badges",
     "value": 6,
     "as_of": "2026-09-03",
     "source_url": "https://gpp-viewer.arisef.org/",
     "verification": "verified",
     "note": "Hub run vision_first_gemini_3_5_flash: idle, 74,636 turns",
     "rank": 3
    }
   ],
   "flags": [],
   "sources": [
    "https://gpp-viewer.arisef.org/",
    "https://github.com/waylaidwanderer/gemini-plays-pokemon-public/branches/active",
    "https://www.reddit.com/r/ClaudePlaysPokemon/"
   ],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "auto_flags": [
    "A Reddit post announced a 3.7 vs 3.5 Flash race. We could not read it (Reddit blocked our fetch), so we do not claim these hub runs are that race."
   ],
   "section": "pokemon",
   "config": {
    "harness": "Gemini Plays Pokémon harness, 'vision-first' setup",
    "inputs": "vision-first (exact inputs not documented)",
    "tools": "custom subagents, Python sandbox, notepads",
    "assistance": "not disclosed",
    "budget": "not disclosed"
   },
   "creator": {
    "name": "Gemini Plays Pokémon",
    "links": [
     {
      "label": "Web viewer & replays",
      "url": "https://gpp-viewer.arisef.org/"
     },
     {
      "label": "Public run repo",
      "url": "https://github.com/waylaidwanderer/gemini-plays-pokemon-public"
     }
    ]
   },
   "url": "https://aivsgames.com/games/pokemon-blue/#gemini-flash-race-pokemon-blue"
  },
  {
   "id": "gemini-plays-pokemon-emerald",
   "kind": "run",
   "game": "pokemon-emerald",
   "game_name": "Pokémon Emerald",
   "title": "Gemini Plays Pokémon: Emerald (Gemini 4 Argon & Gemini 3.8 Flash)",
   "status": "LIVE",
   "summary": "Two Gemini models are playing Pokémon Emerald on the Gemini Plays Pokémon hub. Gemini 4 Argon has all 8 badges after 3,797 turns, and Gemini 3.8 Flash has 5 badges after 9,274 turns. Gemini 3.8 Flash was last seen at Fortree City Gym.",
   "fetcher": "gpp_viewer",
   "results": [
    {
     "model": "Gemini 4 Argon",
     "metric": "Badges",
     "value": 8,
     "as_of": "2026-10-18",
     "source_url": "https://gpp-viewer.arisef.org/",
     "verification": "verified",
     "note": "Out of 8 · hub run emerald_gemini_4_argon (online)"
    },
    {
     "model": "Gemini 4 Argon",
     "metric": "Turns",
     "value": 3797,
     "as_of": "2026-10-18",
     "source_url": "https://gpp-viewer.arisef.org/",
     "verification": "verified"
    },
    {
     "model": "Gemini 3.8 Flash",
     "metric": "Badges",
     "value": 5,
     "as_of": "2026-10-18",
     "source_url": "https://gpp-viewer.arisef.org/",
     "verification": "verified",
     "note": "Out of 8 · hub run emerald_gemini_3_8_flash (online)"
    },
    {
     "model": "Gemini 3.8 Flash",
     "metric": "Turns",
     "value": 9274,
     "as_of": "2026-10-18",
     "source_url": "https://gpp-viewer.arisef.org/",
     "verification": "verified"
    }
   ],
   "sources": [
    "https://gpp-viewer.arisef.org/",
    "https://github.com/waylaidwanderer/gemini-plays-pokemon-public",
    "https://www.twitch.tv/gemini_plays_pokemon"
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "current_goal": "Defeat Gym Leader Winona for Gym 6 (Feather Badge)",
   "hub_runs": [
    {
     "run_id": "emerald_gemini_3_8_flash",
     "status": "online",
     "badges": 5,
     "turns": 9274,
     "started": "2026-10-13",
     "last_active": "2026-10-18"
    },
    {
     "run_id": "emerald_gemini_4_argon",
     "status": "online",
     "badges": 8,
     "turns": 3797,
     "started": "2026-10-13",
     "last_active": "2026-10-18"
    },
    {
     "run_id": "blue_gemini_3_8_flash",
     "status": "finished",
     "badges": 3,
     "turns": 5648,
     "started": "2026-10-06",
     "last_active": "2026-10-14"
    },
    {
     "run_id": "sors_gemini_3_8_flash",
     "status": "idle",
     "badges": 8,
     "turns": 38783,
     "started": "2026-09-27",
     "last_active": "2026-10-13"
    },
    {
     "run_id": "blue_gemini_4_argon",
     "status": "finished",
     "badges": 8,
     "turns": 1753,
     "started": "2026-10-10",
     "last_active": "2026-10-11"
    },
    {
     "run_id": "vision_first_gemini_3_8_flash",
     "status": "idle",
     "badges": 8,
     "turns": 47889,
     "started": "2026-09-03",
     "last_active": "2026-09-27"
    },
    {
     "run_id": "vision_first_gemini_3_7_flash",
     "status": "idle",
     "badges": 8,
     "turns": 55601,
     "started": "2026-08-16",
     "last_active": "2026-09-19"
    },
    {
     "run_id": "vision_first_gemini_3_5_flash",
     "status": "idle",
     "badges": 6,
     "turns": 74636,
     "started": "2026-08-16",
     "last_active": "2026-09-03"
    }
   ],
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "section": "pokemon",
   "config": {
    "harness": "Gemini Plays Pokémon harness (mGBA emulator)",
    "inputs": "emulator video frames + game-state analysis",
    "tools": "custom subagents, Python sandbox, notepads (all public on GitHub)",
    "assistance": "not disclosed",
    "budget": "not disclosed"
   },
   "creator": {
    "name": "Gemini Plays Pokémon",
    "links": [
     {
      "label": "Twitch channel",
      "url": "https://www.twitch.tv/gemini_plays_pokemon"
     },
     {
      "label": "Web viewer & replays",
      "url": "https://gpp-viewer.arisef.org/"
     },
     {
      "label": "Public run repo",
      "url": "https://github.com/waylaidwanderer/gemini-plays-pokemon-public"
     }
    ]
   },
   "progress_multi": [
    {
     "label": "Gemini 4 Argon",
     "current": 8,
     "max": 8
    },
    {
     "label": "Gemini 3.8 Flash",
     "current": 5,
     "max": 8
    }
   ],
   "stream": "live",
   "url": "https://aivsgames.com/games/pokemon-emerald/#gemini-plays-pokemon-emerald"
  },
  {
   "id": "gpt-plays-pokemon",
   "kind": "run",
   "game": "pokemon",
   "game_name": "Pokémon (main series)",
   "title": "GPT Plays Pokémon (vision-only harness)",
   "status": "PAUSED",
   "summary": "The creator posted that GPT-6 Astra (high) became Champion in 18h 12m, against 96h 35m for GPT-5.6 Sol (max), and announced a fresh run for the Twitch channel. We could not confirm that a stream is live now.",
   "fetcher": "twitch_status",
   "twitch_login": "gpt_plays_pokemon",
   "results": [
    {
     "model": "GPT-6 Astra",
     "metric": "Time to Champion",
     "value": "18h 12m",
     "as_of": "2026-09-05",
     "source_url": "https://www.unrollnow.com/status/2095596013168050551",
     "verification": "self-reported",
     "note": "Creator thread (@Clad3815). Game version not stated in the thread."
    },
    {
     "model": "GPT-5.6 Sol",
     "metric": "Time to Champion",
     "value": "96h 35m",
     "as_of": "2026-09-05",
     "source_url": "https://www.unrollnow.com/status/2095596013168050551",
     "verification": "conflicting",
     "note": "Conflicts with 153h51 on BenchmarkList (Emerald). The two may be different runs."
    },
    {
     "model": "GPT-6 Astra",
     "metric": "Live now",
     "value": "no",
     "as_of": "2026-10-18",
     "source_url": "https://www.twitch.tv/gpt_plays_pokemon",
     "verification": "verified",
     "note": "Checked via a public uptime endpoint (decapi.me). Move to the official Twitch API once a dev app exists."
    }
   ],
   "flags": [
    "Live status unverified. TwitchMetrics shows streams only up to Jul 2026."
   ],
   "sources": [
    "https://www.twitch.tv/gpt_plays_pokemon",
    "https://www.unrollnow.com/status/2095596013168050551",
    "https://www.twitchmetrics.net/c/1314505924-gpt_plays_pokemon/streams"
   ],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "Twitch: waiting for an official Twitch API app. No scraping. Last known status kept with its date."
   },
   "section": "pokemon",
   "config": {
    "harness": "GPT Plays Pokémon (vision-only)",
    "inputs": "screen only (vision-only)",
    "tools": "Codex App Server + custom tools (per BenchmarkList)",
    "assistance": "not disclosed",
    "budget": "effort level named per run (e.g. high, max)"
   },
   "creator": {
    "name": "GPT Plays Pokémon",
    "links": [
     {
      "label": "Twitch channel",
      "url": "https://www.twitch.tv/gpt_plays_pokemon"
     }
    ]
   },
   "url": "https://aivsgames.com/games/pokemon/#gpt-plays-pokemon"
  },
  {
   "id": "kaggle-chess",
   "kind": "leaderboard",
   "game": "chess",
   "game_name": "Chess",
   "title": "Kaggle Game Arena: Chess (text)",
   "status": "RECENT",
   "summary": "Models play chess against each other with no engine help and are ranked by Elo (Bradley–Terry). Each pair of models plays 40 games.",
   "fetcher": "kaggle",
   "results": [
    {
     "model": "Gemini 3.8 Flash",
     "metric": "Elo",
     "value": 1452,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 1,
     "note": "Cross-checked 2026-10-18 13:35 EDT: The Aggregate's daily snapshot (2026-10-18) lists 1451.67. Kaggle's Oct 8 version (v2) showed 1429."
    },
    {
     "model": "GPT-6 Astra",
     "metric": "Elo",
     "value": 1421,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 2
    },
    {
     "model": "Gemini 3.1 Pro Preview",
     "metric": "Elo",
     "value": 1315,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 3
    },
    {
     "model": "GPT-5.5",
     "metric": "Elo",
     "value": 1246,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 4
    },
    {
     "model": "Gemini 3.6 Flash",
     "metric": "Elo",
     "value": 1222,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 5
    },
    {
     "model": "Gemini 3 Flash Preview",
     "metric": "Elo",
     "value": 1211,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 6
    },
    {
     "model": "GPT-6.1 Sol",
     "metric": "Elo",
     "value": 1209,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 7
    },
    {
     "model": "GPT-5.6 Sol",
     "metric": "Elo",
     "value": 1120,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 8
    },
    {
     "model": "GPT-6 Sol",
     "metric": "Elo",
     "value": 1008,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 9
    },
    {
     "model": "Claude Opus 5.5",
     "metric": "Elo",
     "value": 965,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 10
    },
    {
     "model": "GPT-5.4",
     "metric": "Elo",
     "value": 897,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 11
    },
    {
     "model": "Grok 4.7",
     "metric": "Elo",
     "value": 845,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/chess-text",
     "verification": "verified",
     "rank": 12
    }
   ],
   "flags": [
    "The Game Arena hub card shows 1499 for #1, which conflicts with the 1452 on the benchmark page. We use the benchmark page."
   ],
   "sources": [
    "https://www.kaggle.com/benchmarks/kaggle/chess-text",
    "https://www.kaggle.com/game-arena"
   ],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "Kaggle: reviewed snapshot (captured 2026-10-18) until bulk-access permission is confirmed"
   },
   "section": "kaggle",
   "kaggle_slug": "chess-text",
   "config": {
    "harness": "Kaggle Game Arena text harness (FEN/PGN)",
    "inputs": "board as text",
    "tools": "none (no chess engine)",
    "assistance": "none",
    "budget": "40 games per model pair; per-model reasoning settings not shown"
   },
   "auto_flags": [
    "Game Arena hub card showed 1499 for #1 (seen Oct 18, 2026). Not found on the chess-text benchmark page (1452, updated Oct 15) or in The Aggregate's Oct 18 snapshot (1451.67). We use the benchmark page."
   ],
   "url": "https://aivsgames.com/games/chess/#kaggle-chess"
  },
  {
   "id": "kaggle-go",
   "kind": "leaderboard",
   "section": "kaggle",
   "game": "go",
   "game_name": "Go",
   "title": "Kaggle Game Arena: Go",
   "status": "RECENT",
   "fetcher": "kaggle",
   "kaggle_slug": "go",
   "summary": "Models play Go against each other on Kaggle Game Arena and are ranked by a Bayesian skill rating.",
   "config": {
    "harness": "Kaggle Game Arena harness",
    "tools": "none",
    "assistance": "none",
    "inputs": "board as text",
    "budget": "per-model settings not shown"
   },
   "sources": [
    "https://www.kaggle.com/benchmarks/kaggle/go"
   ],
   "results": [
    {
     "model": "GPT-6 Astra",
     "metric": "Elo",
     "value": 1794,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 1
    },
    {
     "model": "Claude Fable 5.1",
     "metric": "Elo",
     "value": 1648,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 2
    },
    {
     "model": "Claude Opus 5",
     "metric": "Elo",
     "value": 1578,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 3
    },
    {
     "model": "Claude Opus 5.5",
     "metric": "Elo",
     "value": 1565,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 4
    },
    {
     "model": "Claude Sonnet 5.5",
     "metric": "Elo",
     "value": 1468,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 5
    },
    {
     "model": "GPT-6.1 Sol",
     "metric": "Elo",
     "value": 1458,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 6
    },
    {
     "model": "Gemini 3.8 Flash",
     "metric": "Elo",
     "value": 1221,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 7
    },
    {
     "model": "Gemini 3.1 Pro Preview",
     "metric": "Elo",
     "value": 1145,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 8
    },
    {
     "model": "Claude Sonnet 5",
     "metric": "Elo",
     "value": 1129,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 9
    },
    {
     "model": "Claude Opus 4.8",
     "metric": "Elo",
     "value": 1095,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 10
    },
    {
     "model": "Claude Haiku 5.5",
     "metric": "Elo",
     "value": 1085,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 11
    },
    {
     "model": "Grok 4.6",
     "metric": "Elo",
     "value": 1018,
     "as_of": "2026-10-15",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/go",
     "verification": "verified",
     "rank": 12
    }
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "Kaggle: reviewed snapshot (captured 2026-10-18) until bulk-access permission is confirmed"
   },
   "url": "https://aivsgames.com/games/go/#kaggle-go"
  },
  {
   "id": "kaggle-poker",
   "kind": "leaderboard",
   "section": "kaggle",
   "game": "poker",
   "game_name": "Heads-up Poker",
   "title": "Kaggle Game Arena: Heads-up Poker",
   "status": "RECENT",
   "fetcher": "kaggle",
   "kaggle_slug": "poker-heads-up",
   "summary": "Heads-up poker tests reasoning with hidden information. Ranked by mean big blinds won per 100 hands (BB/100).",
   "config": {
    "harness": "Kaggle Game Arena harness",
    "tools": "none",
    "assistance": "none",
    "inputs": "cards & betting history as text",
    "budget": "per-model settings not shown"
   },
   "sources": [
    "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up"
   ],
   "results": [
    {
     "model": "GPT-5.6 Sol",
     "metric": "Mean BB/100",
     "value": 30,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 1
    },
    {
     "model": "GPT-5.5",
     "metric": "Mean BB/100",
     "value": 25,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 2
    },
    {
     "model": "GPT-6 Astra",
     "metric": "Mean BB/100",
     "value": 25,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 3
    },
    {
     "model": "Claude Fable 5.1",
     "metric": "Mean BB/100",
     "value": 18,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 4
    },
    {
     "model": "Claude Opus 5",
     "metric": "Mean BB/100",
     "value": 10,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 5
    },
    {
     "model": "GPT-5.6 Terra",
     "metric": "Mean BB/100",
     "value": 7,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 6
    },
    {
     "model": "GPT-5.2",
     "metric": "Mean BB/100",
     "value": 5,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 7
    },
    {
     "model": "GPT-5.6 Luna",
     "metric": "Mean BB/100",
     "value": 5,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 8
    },
    {
     "model": "Gemini 3.8 Flash",
     "metric": "Mean BB/100",
     "value": 5,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 9
    },
    {
     "model": "Gemini 3.6 Flash",
     "metric": "Mean BB/100",
     "value": 3,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 10
    },
    {
     "model": "GPT-5.4",
     "metric": "Mean BB/100",
     "value": 1,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 11
    },
    {
     "model": "Grok 4.6",
     "metric": "Mean BB/100",
     "value": 0,
     "as_of": "2026-10-08",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/poker-heads-up",
     "verification": "verified",
     "rank": 12
    }
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "Kaggle: reviewed snapshot (captured 2026-10-18) until bulk-access permission is confirmed"
   },
   "url": "https://aivsgames.com/games/poker/#kaggle-poker"
  },
  {
   "id": "kaggle-werewolf",
   "kind": "leaderboard",
   "section": "kaggle",
   "game": "werewolf",
   "game_name": "Werewolf",
   "title": "Kaggle Game Arena: Werewolf",
   "status": "ARCHIVE",
   "fetcher": "kaggle",
   "kaggle_slug": "werewolf",
   "summary": "Eight-player Werewolf tests persuasion, deception and teamwork. Ranked by equilibrium rating.",
   "config": {
    "harness": "Kaggle Game Arena harness",
    "tools": "none",
    "assistance": "none",
    "inputs": "game chat & role info as text",
    "budget": "per-model settings not shown"
   },
   "sources": [
    "https://www.kaggle.com/benchmarks/kaggle/werewolf"
   ],
   "results": [
    {
     "model": "GPT-5.4",
     "metric": "Equilibrium Rating",
     "value": "0.08%",
     "as_of": "2026-06-12",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/werewolf",
     "verification": "verified",
     "rank": 1
    },
    {
     "model": "Gemini 3.1 Pro Preview",
     "metric": "Equilibrium Rating",
     "value": "-0.38%",
     "as_of": "2026-06-12",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/werewolf",
     "verification": "verified",
     "rank": 2
    },
    {
     "model": "Gemini 3 Flash Preview",
     "metric": "Equilibrium Rating",
     "value": "-1.15%",
     "as_of": "2026-06-12",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/werewolf",
     "verification": "verified",
     "rank": 3
    },
    {
     "model": "Claude Opus 4.6",
     "metric": "Equilibrium Rating",
     "value": "-1.25%",
     "as_of": "2026-06-12",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/werewolf",
     "verification": "verified",
     "rank": 4
    },
    {
     "model": "Claude Sonnet 4.6",
     "metric": "Equilibrium Rating",
     "value": "-2.15%",
     "as_of": "2026-06-12",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/werewolf",
     "verification": "verified",
     "rank": 5
    },
    {
     "model": "GPT-5.4 mini",
     "metric": "Equilibrium Rating",
     "value": "-5.55%",
     "as_of": "2026-06-12",
     "source_url": "https://www.kaggle.com/benchmarks/kaggle/werewolf",
     "verification": "verified",
     "rank": 6
    }
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "Kaggle: reviewed snapshot (captured 2026-10-18) until bulk-access permission is confirmed"
   },
   "url": "https://aivsgames.com/games/werewolf/#kaggle-werewolf"
  },
  {
   "id": "kradleverse-minecraft",
   "kind": "leaderboard",
   "game": "minecraft",
   "game_name": "Minecraft",
   "title": "KradleVerse: AI agents play Minecraft",
   "status": "LIVE",
   "summary": "A live arena where community AI agents compete in Minecraft, ranked by 30-day win rate (minimum 10 runs).",
   "fetcher": "kradleverse",
   "results": [
    {
     "model": "claude-opus-4-6 (agent: Echo-Sinclair, OpenClaw)",
     "metric": "30-day win %",
     "value": "28%",
     "as_of": "2026-10-18",
     "source_url": "https://kradleverse.com/leaderboard",
     "verification": "verified",
     "note": "15 wins",
     "rank": 1
    }
   ],
   "sources": [
    "https://kradleverse.com/leaderboard"
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "section": "minecraft",
   "config": {
    "harness": "community agents (framework shown per row)",
    "inputs": "not disclosed",
    "tools": "not disclosed",
    "assistance": "not disclosed",
    "budget": "ranked on last 30 days, 10+ runs to qualify"
   },
   "creator": {
    "name": "KradleVerse",
    "links": [
     {
      "label": "Watch live matches",
      "url": "https://kradleverse.com/"
     }
    ]
   },
   "url": "https://aivsgames.com/games/minecraft/#kradleverse-minecraft"
  },
  {
   "id": "orak",
   "kind": "leaderboard",
   "game": "multi-game",
   "game_name": "Multi-game (Orak: 12 games)",
   "title": "Orak (KRAFTON AI): 12 video games",
   "status": "ARCHIVE",
   "summary": "Covers Street Fighter III, Super Mario, Ace Attorney, Pokémon, Minecraft, Stardew Valley, StarCraft II, Slay the Spire and more. Last updated Jun 9, 2025.",
   "fetcher": null,
   "results": [
    {
     "model": "Gemini 2.5 Pro (preview 03-25)",
     "metric": "Avg rank (lower is better)",
     "value": 3.9,
     "as_of": "2025-06-09",
     "source_url": "https://krafton-ai.github.io/orak-leaderboard/",
     "verification": "verified",
     "rank": 1
    },
    {
     "model": "GPT-4o (2024-08-06)",
     "metric": "Avg rank (lower is better)",
     "value": 4.3,
     "as_of": "2025-06-09",
     "source_url": "https://krafton-ai.github.io/orak-leaderboard/",
     "verification": "verified",
     "rank": 2
    },
    {
     "model": "o3-mini",
     "metric": "Avg rank (lower is better)",
     "value": 4.6,
     "as_of": "2025-06-09",
     "source_url": "https://krafton-ai.github.io/orak-leaderboard/",
     "verification": "verified",
     "rank": 3
    }
   ],
   "sources": [
    "https://krafton-ai.github.io/orak-leaderboard/"
   ],
   "flags": [],
   "last_checked": "2026-10-17",
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "manual source"
   },
   "section": "more",
   "config": {
    "harness": "Orak benchmark harness",
    "inputs": "per game",
    "tools": "not disclosed",
    "assistance": "not disclosed",
    "budget": "not disclosed"
   },
   "url": "https://aivsgames.com/games/multi-game/#orak"
  },
  {
   "id": "pokeagent-speedrun",
   "kind": "leaderboard",
   "section": "pokemon",
   "game": "pokemon-emerald",
   "game_name": "Pokémon Emerald",
   "title": "PokéAgent Challenge: Speedrun track (NeurIPS 2025 final)",
   "status": "ARCHIVE",
   "fetcher": null,
   "benchmark_version": "NeurIPS 2025, scored to first gym (Roxanne)",
   "summary": "Agents race through Pokémon Emerald in real time; the official NeurIPS 2025 evaluation ended at the first gym. 22 teams submitted and 6 completed all 15 milestones. Ranked by wall-clock time; action counts are reported separately.",
   "config": {
    "harness": "PokéAgent Emerald environment (real-time)",
    "inputs": "per team (paper appendix)",
    "tools": "per team: RL, LLM harnesses, hybrids",
    "assistance": "none during evaluation",
    "budget": "wall-clock time + action count"
   },
   "sources": [
    "https://arxiv.org/abs/2603.15563",
    "https://pokeagentchallenge.com/speedrunning.html"
   ],
   "flags": [
    "Underlying models per team are not named in our sources. Entrants are listed by team."
   ],
   "results": [
    {
     "model": "Heatz (team)",
     "metric": "Time to first gym",
     "value": "40:13",
     "as_of": "2026-03-16",
     "source_url": "https://arxiv.org/abs/2603.15563",
     "verification": "verified",
     "note": "1,608 actions · Scripted Policy Distillation (LLM-written policies distilled + RL)",
     "rank": 1
    },
    {
     "model": "Hamburg PokéRunners (team)",
     "metric": "Time to first gym",
     "value": "1:14:43",
     "as_of": "2026-03-16",
     "source_url": "https://arxiv.org/abs/2603.15563",
     "verification": "verified",
     "note": "recurrent PPO (pure RL)",
     "rank": 2
    },
    {
     "model": "anthonys (team)",
     "metric": "Time to first gym",
     "value": "1:29:17",
     "as_of": "2026-03-16",
     "source_url": "https://arxiv.org/abs/2603.15563",
     "verification": "verified",
     "note": "tool-calling VLM policy",
     "rank": 3
    },
    {
     "model": "Deepest (team)",
     "metric": "Time to first gym",
     "value": null,
     "as_of": "2026-03-16",
     "source_url": "https://arxiv.org/abs/2603.15563",
     "verification": "verified",
     "note": "5th by time (exact time not in our source) · fewest actions: 649 · Judge's Choice",
     "rank": 5
    }
   ],
   "fetch_status": {
    "ok": null,
    "checked": "2026-10-18T16:31:30-04:00",
    "note": "manual source"
   },
   "url": "https://aivsgames.com/games/pokemon-emerald/#pokeagent-speedrun"
  },
  {
   "id": "pokemon-autonomous-completion",
   "kind": "leaderboard",
   "game": "pokemon",
   "game_name": "Pokémon (main series)",
   "title": "Pokémon Autonomous Completion (BenchmarkList)",
   "status": "RECENT",
   "summary": "Self-reported vision-only runs that played main-series Pokémon games through to the end.",
   "fetcher": "benchmarklist_pokemon",
   "results": [
    {
     "model": "GPT-5.6 Sol",
     "metric": "Time to Champion (Emerald)",
     "value": "153h51",
     "as_of": "2026-07-21",
     "source_url": "https://benchmarklist.com/benchmarks/pokemon_autonomous_completion/",
     "verification": "conflicting",
     "note": "8 badges. Conflicts with the creator's 96h 35m claim.",
     "rank": 1
    },
    {
     "model": "GPT-5",
     "metric": "Time to Champion (Red)",
     "value": "161:57:00",
     "as_of": "2025-08-14",
     "source_url": "https://benchmarklist.com/benchmarks/pokemon_autonomous_completion/",
     "verification": "self-reported",
     "note": "6,470 steps, 8 badges",
     "rank": 2
    },
    {
     "model": "GPT-5",
     "metric": "Time to Champion (Crystal)",
     "value": "202:02:12",
     "as_of": "2025-08-25",
     "source_url": "https://benchmarklist.com/benchmarks/pokemon_autonomous_completion/",
     "verification": "self-reported",
     "note": "9,517 steps, 16 badges",
     "rank": 3
    }
   ],
   "sources": [
    "https://benchmarklist.com/benchmarks/pokemon_autonomous_completion/"
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": false,
    "checked": "2026-10-18T16:31:30-04:00",
    "error": "403 Client Error: Forbidden for url: https://benchmarklist.com/benchmarks/pokemon_autonomous_completion/",
    "note": "kept last good data"
   },
   "section": "pokemon",
   "config": {
    "harness": "Codex App Server + custom tools",
    "inputs": "vision-only",
    "tools": "custom tools (not itemised)",
    "assistance": "not disclosed",
    "budget": "not disclosed"
   },
   "url": "https://aivsgames.com/games/pokemon/#pokemon-autonomous-completion"
  },
  {
   "id": "pokemon-vgc-bench",
   "kind": "leaderboard",
   "game": "pokemon-vgc",
   "game_name": "Pokémon VGC (competitive doubles)",
   "title": "Pokémon VGC Bench",
   "status": "RECENT",
   "summary": "16 AI systems are rated with Bradley–Terry ratings (centered at 1500) across 108 best-of-three series and 247 battles.",
   "fetcher": "vgc_bench",
   "results": [
    {
     "model": "GPT-6 Astra",
     "metric": "Rating",
     "value": 1901.0,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 1
    },
    {
     "model": "GPT-6.1 Sol",
     "metric": "Rating",
     "value": 1846.8,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 2
    },
    {
     "model": "Gemini 3.8 Flash",
     "metric": "Rating",
     "value": 1744.2,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 3
    },
    {
     "model": "GPT-6 Sol",
     "metric": "Rating",
     "value": 1640.5,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 4
    },
    {
     "model": "Claude Fable 5.1",
     "metric": "Rating",
     "value": 1614.4,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 5
    },
    {
     "model": "Claude Opus 5.5",
     "metric": "Rating",
     "value": 1614.4,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 6
    },
    {
     "model": "Muse Spark 1.3",
     "metric": "Rating",
     "value": 1614.4,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 7
    },
    {
     "model": "GPT-5.6 Sol",
     "metric": "Rating",
     "value": 1565.5,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 8
    },
    {
     "model": "GLM 5.3",
     "metric": "Rating",
     "value": 1482.4,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 9
    },
    {
     "model": "Grok 4.7",
     "metric": "Rating",
     "value": 1482.4,
     "as_of": "2026-10-04",
     "source_url": "https://ricotoma.com/pokemon-bench",
     "verification": "verified",
     "rank": 10
    }
   ],
   "sources": [
    "https://ricotoma.com/pokemon-bench"
   ],
   "flags": [],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "section": "pokemon",
   "config": {
    "harness": "Pokémon VGC Bench battle harness",
    "inputs": "battle state (text)",
    "tools": "not disclosed",
    "assistance": "none disclosed",
    "budget": "not disclosed"
   },
   "url": "https://aivsgames.com/games/pokemon-vgc/#pokemon-vgc-bench"
  },
  {
   "id": "videogamebench-lite",
   "kind": "leaderboard",
   "section": "more",
   "game": "videogamebench-suite",
   "game_name": "VideoGameBench Lite suite (3 games)",
   "title": "VideoGameBench Lite (game pauses while the model thinks)",
   "status": "ARCHIVE",
   "fetcher": "vgbench",
   "vgb_table": 1,
   "benchmark_version": "Lite, paused",
   "summary": "The same benchmark with the game paused while the model thinks. Scores are not comparable with the real-time table.",
   "config": {
    "harness": "VG-Agent (VideoGameBench reference agent)",
    "inputs": "raw game frames + high-level objective and controls",
    "tools": "none",
    "assistance": "none",
    "budget": "paused between actions (Lite)"
   },
   "sources": [
    "https://www.vgbench.com/",
    "https://arxiv.org/abs/2505.18134"
   ],
   "flags": [
    "The overall score is the benchmark's own average across its game suite. It is not an AI vs Games ranking."
   ],
   "results": [
    {
     "model": "gpt-4o-2024-08-06",
     "metric": "Overall score",
     "value": "1.6%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + GPT-4o · best game: Kirby's Dream Land 4.8%",
     "rank": 1
    },
    {
     "model": "claude-3-7-sonnet-20250219",
     "metric": "Overall score",
     "value": "1.6%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Sonnet 3.7 · best game: Kirby's Dream Land 4.8%",
     "rank": 1
    },
    {
     "model": "gemini-2.5-pro-preview-03-25",
     "metric": "Overall score",
     "value": "1.6%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Gemini 2.5 Pro · best game: Kirby's Dream Land 4.8%",
     "rank": 1
    },
    {
     "model": "llama-4-maverick-17B-128E-Instruct-FP8",
     "metric": "Overall score",
     "value": "0%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Llama 4 Maverick · 0% on every game",
     "rank": 4
    },
    {
     "model": "gemini-2.0-flash",
     "metric": "Overall score",
     "value": "0%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Gemini 2.0 Flash · 0% on every game",
     "rank": 4
    }
   ],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "url": "https://aivsgames.com/games/videogamebench-suite/#videogamebench-lite"
  },
  {
   "id": "videogamebench",
   "kind": "leaderboard",
   "section": "more",
   "game": "videogamebench-suite",
   "game_name": "VideoGameBench suite (10 test games)",
   "title": "VideoGameBench (real-time)",
   "status": "ARCHIVE",
   "fetcher": "vgbench",
   "vgb_table": 0,
   "benchmark_version": "test set, real-time",
   "summary": "Can vision-language models finish classic games from raw frames alone? The game keeps running while the model thinks. The overall score averages progress across 10 test games, including Pokémon Crystal.",
   "config": {
    "harness": "VG-Agent (VideoGameBench reference agent)",
    "inputs": "raw game frames + high-level objective and controls",
    "tools": "none",
    "assistance": "none",
    "budget": "real-time (game does not pause)"
   },
   "sources": [
    "https://www.vgbench.com/",
    "https://arxiv.org/abs/2505.18134"
   ],
   "flags": [
    "The overall score is the benchmark's own average across its game suite. It is not an AI vs Games ranking."
   ],
   "results": [
    {
     "model": "gemini-2.5-pro-preview-03-25",
     "metric": "Overall score",
     "value": "0.48%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Gemini 2.5 Pro · best game: Kirby's Dream Land (DX) 4.8%",
     "rank": 1
    },
    {
     "model": "gpt-4o-2024-08-06",
     "metric": "Overall score",
     "value": "0.09%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + GPT-4o · best game: Pokemon Crystal 0.9%",
     "rank": 2
    },
    {
     "model": "claude-3-7-sonnet-20250219",
     "metric": "Overall score",
     "value": "0%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Sonnet 3.7 · 0% on every game",
     "rank": 3
    },
    {
     "model": "llama-4-maverick-17B-128E-Instruct-FP8",
     "metric": "Overall score",
     "value": "0%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Llama 4 Maverick · 0% on every game",
     "rank": 3
    },
    {
     "model": "gemini-2.0-flash",
     "metric": "Overall score",
     "value": "0%",
     "as_of": "2025-05-23",
     "source_url": "https://www.vgbench.com/",
     "verification": "verified",
     "note": "listed as VG-Agent + Gemini 2.0 Flash · 0% on every game",
     "rank": 3
    }
   ],
   "last_checked": "2026-10-18",
   "fetch_status": {
    "ok": true,
    "checked": "2026-10-18T16:31:30-04:00"
   },
   "url": "https://aivsgames.com/games/videogamebench-suite/#videogamebench"
  }
 ]
}