{
  "schemaVersion": 1,
  "benchmark": {
    "id": "deep-swe",
    "name": "DeepSWE v1.1 · AA evaluation",
    "version": "1.1",
    "category": "coding",
    "question": "Can an agent complete long software changes?",
    "summary": "Long-horizon software engineering, measured inside Artificial Analysis’s coding-agent evaluation.",
    "source": {
      "name": "Artificial Analysis",
      "url": "https://artificialanalysis.ai/agents/coding-agents/"
    },
    "measure": "Success on software engineering tasks with automated code verification.",
    "comparisonRule": "Keep this Artificial Analysis cohort separate from DataCurve’s direct mini-swe-agent leaderboard and other harnesses.",
    "limitations": [
      "Cost and time describe the AA coding suite, not the isolated DeepSWE task set.",
      "The checked source does not report per-configuration uncertainty."
    ],
    "coverage": "charted",
    "tags": [
      "coding",
      "software engineering",
      "repository",
      "long horizon",
      "agent"
    ]
  },
  "dataset": {
    "benchmarkId": "deep-swe",
    "version": "1.1",
    "score": {
      "label": "DeepSWE v1.1",
      "unit": "%",
      "direction": "higher",
      "minimum": 0,
      "maximum": 100
    },
    "source": {
      "name": "Artificial Analysis",
      "url": "https://artificialanalysis.ai/agents/coding-agents/",
      "retrievedAt": "2026-09-25T14:39:44.751Z"
    },
    "evidenceLabel": "Independent evaluation",
    "configurationLabel": "Model, coding agent, and effort",
    "comparabilityNote": "Each point is an Artificial Analysis coding-agent configuration. Cost covers the source’s coding suite, not an isolated run of the selected benchmark. Component results stay separate from standalone cohorts published by other evaluators.",
    "costLabel": "USD per task across the AA coding suite",
    "points": [
      {
        "id": "d1003684d8f29b95a3b14d2a7722221a",
        "label": "Gemini 3.8 Flash · Antigravity SDK · high",
        "model": "Gemini 3.8 Flash",
        "provider": "Google",
        "harness": "Antigravity SDK",
        "effort": "high",
        "score": 65.78171091445431,
        "costUsd": 2.4673922004125437,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "11.7 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "13,511,073"
          }
        ]
      },
      {
        "id": "29a1cdad0d140781d05b9b4746cfdd0d",
        "label": "Fable 5.1 (with fallback) · Claude Code · max",
        "model": "Fable 5.1 (with fallback)",
        "provider": "Anthropic",
        "harness": "Claude Code",
        "effort": "max",
        "score": 64.30678466076701,
        "costUsd": 12.388092375137534,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "34.8 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "5,715,254"
          }
        ]
      },
      {
        "id": "409518818f1e9ad21995d70631ab05b6",
        "label": "Opus 5 · Claude Code · max",
        "model": "Opus 5",
        "provider": "Anthropic",
        "harness": "Claude Code",
        "effort": "max",
        "score": 62.536873156342196,
        "costUsd": 10.789154699257413,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "41.9 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "11,374,850"
          }
        ]
      },
      {
        "id": "ab5074e404be48bbd4ac3c488ad9a3e2",
        "label": "Opus 5.5 · Claude Code · max",
        "model": "Opus 5.5",
        "provider": "Anthropic",
        "harness": "Claude Code",
        "effort": "max",
        "score": 68.4365781710915,
        "costUsd": 13.036383418426828,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "64.5 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "15,551,383"
          }
        ]
      },
      {
        "id": "cd4c865dc2f539b650d741fd2ebf7f2d",
        "label": "Qwen3.8 Max · Claude Code · default",
        "model": "Qwen3.8 Max",
        "provider": "Alibaba Cloud",
        "harness": "Claude Code",
        "effort": "default",
        "score": 51.0324483775811,
        "costUsd": 3.478965577837015,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "63.1 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "8,706,894"
          }
        ]
      },
      {
        "id": "28ef65135f08f035dc93a97d324fea73",
        "label": "DeepSeek V4 Flash 0731 · Codex · max",
        "model": "DeepSeek V4 Flash 0731",
        "provider": "DeepSeek",
        "harness": "Codex",
        "effort": "max",
        "score": 54.2772861356932,
        "costUsd": 0.08506419165060494,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "18.3 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "15,350,225"
          }
        ]
      },
      {
        "id": "542590ecc1aa12a718d158565d464daa",
        "label": "DeepSeek V4 Pro 0813 · Codex · max",
        "model": "DeepSeek V4 Pro 0813",
        "provider": "DeepSeek",
        "harness": "Codex",
        "effort": "max",
        "score": 57.227138643067796,
        "costUsd": 0.23804610720792058,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "40.3 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "24,180,108"
          }
        ]
      },
      {
        "id": "b3c56ec69185d544992fd578edb75e8c",
        "label": "GPT-5.6 Luna · Codex · max",
        "model": "GPT-5.6 Luna",
        "provider": "OpenAI",
        "harness": "Codex",
        "effort": "max",
        "score": 66.3716814159292,
        "costUsd": 0.43802256393839384,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "23.5 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "14,854,525"
          }
        ]
      },
      {
        "id": "709f801275dedae5675b1f56e97c9df3",
        "label": "GPT-5.6 Sol · Codex · max",
        "model": "GPT-5.6 Sol",
        "provider": "OpenAI",
        "harness": "Codex",
        "effort": "max",
        "score": 72.27138643067849,
        "costUsd": 6.353318230693071,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "20.6 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "10,170,137"
          }
        ]
      },
      {
        "id": "9e810ff3c6cce5ebaaff7c744c687a47",
        "label": "GPT-6 Astra · Codex · max",
        "model": "GPT-6 Astra",
        "provider": "OpenAI",
        "harness": "Codex",
        "effort": "max",
        "score": 67.55162241887909,
        "costUsd": 7.46709735138597,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "29.4 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "3,346,611"
          }
        ]
      },
      {
        "id": "ac8276cd5d15ac2daacb9956ad44c132",
        "label": "GPT-6 Luna · Codex · max",
        "model": "GPT-6 Luna",
        "provider": "OpenAI",
        "harness": "Codex",
        "effort": "max",
        "score": 63.716814159292,
        "costUsd": 0.17591172304455457,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "21.4 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "10,218,069"
          }
        ]
      },
      {
        "id": "7a0dfd2a7a9d93056f860d4675586d66",
        "label": "GPT-6 Sol · Codex · max",
        "model": "GPT-6 Sol",
        "provider": "OpenAI",
        "harness": "Codex",
        "effort": "max",
        "score": 69.0265486725664,
        "costUsd": 2.989729004180416,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "22.3 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "9,840,778"
          }
        ]
      },
      {
        "id": "34970c963bdfd318e46205514d46ea4f",
        "label": "Claude Fable 5.1 XHigh + SWE-2 Medium · Devin Fusion CLI · default",
        "model": "Claude Fable 5.1 XHigh + SWE-2 Medium",
        "provider": "Cognition",
        "harness": "Devin Fusion CLI",
        "effort": "default",
        "score": 63.1268436578171,
        "costUsd": 7.897126985217273,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "35.8 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "9,691,548"
          }
        ]
      },
      {
        "id": "fa2595093fa640e73d072e7d1aa7df82",
        "label": "GPT-6 Astra XHigh + SWE-2 Medium · Devin Fusion CLI · default",
        "model": "GPT-6 Astra XHigh + SWE-2 Medium",
        "provider": "Cognition",
        "harness": "Devin Fusion CLI",
        "effort": "default",
        "score": 67.2566371681416,
        "costUsd": 4.536492777821779,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "24.7 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "6,074,134"
          }
        ]
      },
      {
        "id": "70a5d300021b9986f77d4198272ef1a5",
        "label": "Grok 4.6 · Grok Build · xhigh",
        "model": "Grok 4.6",
        "provider": "xAI",
        "harness": "Grok Build",
        "effort": "xhigh",
        "score": 64.8967551622419,
        "costUsd": 3.5701432805280535,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "19.5 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "5,543,625"
          }
        ]
      },
      {
        "id": "f927d9c41b5c3021996eb14c3dd1341e",
        "label": "Grok 4.7 · Grok Build · xhigh",
        "model": "Grok 4.7",
        "provider": "xAI",
        "harness": "Grok Build",
        "effort": "xhigh",
        "score": 72.5663716814159,
        "costUsd": 8.817468730473037,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "39.2 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "14,324,724"
          }
        ]
      },
      {
        "id": "55b029d6971b4d378335bb168eb98b3b",
        "label": "Kimi K3 · Kimi Code CLI · default",
        "model": "Kimi K3",
        "provider": "Moonshot AI",
        "harness": "Kimi Code CLI",
        "effort": "default",
        "score": 68.4365781710915,
        "costUsd": 5.047195443300332,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "61.5 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "8,481,122"
          }
        ]
      },
      {
        "id": "cd60470c35fd626a116af80fd5915dfa",
        "label": "Muse Spark 1.3 · Muse Code · xhigh",
        "model": "Muse Spark 1.3",
        "provider": "Meta",
        "harness": "Muse Code",
        "effort": "xhigh",
        "score": 73.1563421828909,
        "costUsd": 3.467754041474145,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "18.3 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "16,237,094"
          }
        ]
      },
      {
        "id": "f2de07373c3626f1b9269a6ecad7804c",
        "label": "Muse Spark 1.3 · Muse Code · max",
        "model": "Muse Spark 1.3",
        "provider": "Meta",
        "harness": "Muse Code",
        "effort": "max",
        "score": 71.6814159292036,
        "costUsd": 3.977504096947204,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "18.4 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "16,709,445"
          }
        ]
      },
      {
        "id": "45493c2782f1bd94f6d30d4911f317a0",
        "label": "GLM-5.3 · Opencode · default",
        "model": "GLM-5.3",
        "provider": "Z.ai",
        "harness": "Opencode",
        "effort": "default",
        "score": 61.35693215339229,
        "costUsd": 4.241385833751375,
        "uncertainty": null,
        "sourceUrl": "https://artificialanalysis.ai/agents/coding-agents/",
        "details": [
          {
            "label": "Coding-suite time per task",
            "value": "48.1 min"
          },
          {
            "label": "Coding-suite tokens per task",
            "value": "14,276,424"
          }
        ]
      }
    ]
  }
}