{
  "notice": "Please credit \"Is software solved?\" and the sources listed in credits when you reuse these numbers. They are our own aggregates. For original records, go to each source.",
  "id": "coding-benchmarks",
  "name": "AI coding benchmarks",
  "question": "Can AI models do real engineering tasks?",
  "metric": "Best score so far on SWE-bench Verified, the share of real GitHub issues resolved",
  "solvedIf": "Models resolve nearly every real engineering task",
  "value": "83%",
  "change": "+12%",
  "trend": [
    {
      "label": "2024-11",
      "value": 31
    },
    {
      "label": "2024-12",
      "value": 31
    },
    {
      "label": "2025-01",
      "value": 31
    },
    {
      "label": "2025-02",
      "value": 61
    },
    {
      "label": "2025-03",
      "value": 61
    },
    {
      "label": "2025-04",
      "value": 62
    },
    {
      "label": "2025-05",
      "value": 71
    },
    {
      "label": "2025-06",
      "value": 71
    },
    {
      "label": "2025-07",
      "value": 71
    },
    {
      "label": "2025-08",
      "value": 74
    },
    {
      "label": "2025-09",
      "value": 74
    },
    {
      "label": "2025-10",
      "value": 74
    },
    {
      "label": "2025-11",
      "value": 77
    },
    {
      "label": "2025-12",
      "value": 77
    },
    {
      "label": "2026-01",
      "value": 77
    },
    {
      "label": "2026-02",
      "value": 79
    },
    {
      "label": "2026-03",
      "value": 79
    },
    {
      "label": "2026-04",
      "value": 83
    },
    {
      "label": "2026-05",
      "value": 83
    },
    {
      "label": "2026-06",
      "value": 83
    },
    {
      "label": "2026-07",
      "value": 83
    },
    {
      "label": "2026-08",
      "value": 83
    },
    {
      "label": "2026-09",
      "value": 83
    },
    {
      "label": "2026-10",
      "value": 83
    }
  ],
  "trendLabel": "Best SWE-bench Verified score so far, in percent, by model release month.",
  "caution": [
    "A benchmark is not a job. These tasks are fixed in advance and scored automatically, so a high score does not show that a model can run a real software project.",
    "Scores differ widely between benchmarks, and models can be tuned for well-known tests. New models are not always tested right away, so the latest result can lag."
  ],
  "score": 83,
  "scoreReason": "The best model resolves 83% of SWE-bench Verified tasks.",
  "analysis": {
    "title": "Other public benchmarks",
    "intro": "The best score depends heavily on the benchmark, so no single number settles this.",
    "points": [
      "MirrorCode: 79%, from claude-opus-5-5.",
      "DeepSWE: 74%, from gpt-6-astra.",
      "FrontierSWE: 66%, from gpt-6-astra.",
      "FrontierCode: 55%, from claude-opus-5-5.",
      "METR estimates the longest task a model completes half the time at about 17 hours of human work, with a range of 8 to 55 hours."
    ]
  },
  "fromHigh": "At its high",
  "status": "Improving",
  "rule": "Improving when the best score is higher than a year before.",
  "direction": "toward",
  "checks": [
    {
      "date": "2026-11-02",
      "value": "83%",
      "change": "+12%",
      "status": "Improving"
    },
    {
      "date": "2026-11-01",
      "value": "83%",
      "change": "+12%",
      "status": "Improving"
    }
  ],
  "url": "/signals/coding-benchmarks",
  "api": "/api/signals/coding-benchmarks.json",
  "credits": [
    {
      "name": "Epoch AI Benchmarking Hub",
      "url": "https://epoch.ai/benchmarks",
      "signal": "coding-benchmarks",
      "license": "CC BY 4.0",
      "credit": "Epoch AI, 'Capabilities & benchmarking'. Published online at epoch.ai. Retrieved from https://epoch.ai/benchmarks. Results collected from other projects keep their original terms, and we quote only top scores from those."
    },
    {
      "name": "METR Time Horizons",
      "url": "https://metr.org/time-horizons/",
      "signal": "coding-benchmarks",
      "license": "No data license found",
      "credit": "Figure quoted from METR Time Horizons."
    }
  ]
}