{
  "schemaVersion": 1,
  "benchmark": {
    "id": "terminal-bench-4",
    "name": "Terminal-Bench 4",
    "version": "4.0.0",
    "category": "coding",
    "question": "Which agent can finish difficult terminal work?",
    "summary": "Software, systems, CAD, scientific computing, and formal proof tasks executed in a terminal.",
    "source": {
      "name": "Harbor Framework",
      "url": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard",
      "methodologyUrl": "https://github.com/harbor-framework/terminal-bench/releases/tag/v4.0.0"
    },
    "measure": "Task success over 66 tasks and five trials per configuration, with source-reported 95% confidence intervals.",
    "comparisonRule": "aicharts’ primary terminal benchmark. Compare exact version 4.0.0 with the named model, agent version, and effort.",
    "limitations": [
      "An agent and model are evaluated together; this is not a model-only ranking.",
      "Different harnesses and effort settings remain separate configurations.",
      "Cost covers the full 330-trial evaluation. The source does not specify its pricing or confidence-interval method."
    ],
    "coverage": "charted",
    "tags": [
      "terminal",
      "coding",
      "software engineering",
      "agents",
      "computer",
      "standard"
    ]
  },
  "dataset": {
    "benchmarkId": "terminal-bench-4",
    "version": "4.0.0",
    "score": {
      "label": "Task success",
      "unit": "%",
      "direction": "higher",
      "minimum": 0,
      "maximum": 100
    },
    "source": {
      "name": "Harbor Framework",
      "url": "https://hub.harborframework.com/datasets/terminal-bench/terminal-bench/latest?leaderboard=4-0-0&tab=leaderboard",
      "retrievedAt": "2026-09-19T22:19:44.733Z",
      "revision": "c7878bd7cbd05d1dd21586b83eb3a1929a412312"
    },
    "evidenceLabel": "Owner-published submissions",
    "configurationLabel": "Model, agent version, and effort",
    "comparabilityNote": "66 tasks × 5 trials per configuration. Model, agent, and effort all affect results. The owner reports 95% confidence intervals; its interval method and price basis are unspecified.",
    "costLabel": "USD for the full 330-trial evaluation",
    "points": [
      {
        "id": "2026-09-02-anthropic-claude-fable-5-1-max-claude-code",
        "label": "Fable 5.1 · Claude Code · max",
        "model": "Fable 5.1",
        "provider": "Anthropic",
        "harness": "Claude Code 2.1.257",
        "effort": "max",
        "score": 57.88,
        "costUsd": 6243.5,
        "uncertainty": {
          "lower": 54.120000000000005,
          "upper": 61.64,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-09-02-anthropic-claude-fable-5-1-max-claude-code.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "64.9 min"
          }
        ]
      },
      {
        "id": "2026-08-26-anthropic-claude-opus-5-max-claude-code",
        "label": "Opus 5 · Claude Code · max",
        "model": "Opus 5",
        "provider": "Anthropic",
        "harness": "Claude Code 2.1.231",
        "effort": "max",
        "score": 51.82,
        "costUsd": 5969.11,
        "uncertainty": {
          "lower": 48.43,
          "upper": 55.21,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-anthropic-claude-opus-5-max-claude-code.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "79.9 min"
          }
        ]
      },
      {
        "id": "2026-08-26-anthropic-claude-fable-5-max-claude-code",
        "label": "Fable 5 · Claude Code · max",
        "model": "Fable 5",
        "provider": "Anthropic",
        "harness": "Claude Code 2.1.231",
        "effort": "max",
        "score": 44.55,
        "costUsd": 7265.01,
        "uncertainty": {
          "lower": 40.699999999999996,
          "upper": 48.4,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-anthropic-claude-fable-5-max-claude-code.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "70.0 min"
          }
        ]
      },
      {
        "id": "2026-08-26-anthropic-glm-5-3-max-claude-code",
        "label": "GLM-5.3 · Claude Code · max",
        "model": "GLM-5.3",
        "provider": "Z.ai",
        "harness": "Claude Code 2.1.207",
        "effort": "max",
        "score": 41.82,
        "costUsd": 2727.63,
        "uncertainty": {
          "lower": 38.59,
          "upper": 45.05,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-anthropic-glm-5-3-max-claude-code.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "97.2 min"
          }
        ]
      },
      {
        "id": "2026-08-26-openai-gpt-5-6-sol-max-codex",
        "label": "GPT-5.6 Sol · Codex · max",
        "model": "GPT-5.6 Sol",
        "provider": "OpenAI",
        "harness": "Codex 0.149.1",
        "effort": "max",
        "score": 37.27,
        "costUsd": 2541.7,
        "uncertainty": {
          "lower": 33.49,
          "upper": 41.050000000000004,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-openai-gpt-5-6-sol-max-codex.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "39.8 min"
          }
        ]
      },
      {
        "id": "2026-08-26-anthropic-claude-opus-4-8-max-claude-code",
        "label": "Opus 4.8 · Claude Code · max",
        "model": "Opus 4.8",
        "provider": "Anthropic",
        "harness": "Claude Code 2.1.231",
        "effort": "max",
        "score": 23.64,
        "costUsd": 6481.26,
        "uncertainty": {
          "lower": 20.080000000000002,
          "upper": 27.2,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-anthropic-claude-opus-4-8-max-claude-code.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "86.0 min"
          }
        ]
      },
      {
        "id": "2026-08-26-openai-gpt-5-6-terra-max-codex",
        "label": "GPT-5.6 Terra · Codex · max",
        "model": "GPT-5.6 Terra",
        "provider": "OpenAI",
        "harness": "Codex 0.149.1",
        "effort": "max",
        "score": 21.52,
        "costUsd": 1733.52,
        "uncertainty": {
          "lower": 18.27,
          "upper": 24.77,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-openai-gpt-5-6-terra-max-codex.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "41.9 min"
          }
        ]
      },
      {
        "id": "2026-08-26-xai-grok-4-6-none-grok-build",
        "label": "Grok 4.6 · Grok Build · none",
        "model": "Grok 4.6",
        "provider": "xAI",
        "harness": "Grok Build 1.0.5",
        "effort": "none",
        "score": 20.3,
        "costUsd": 3591.58,
        "uncertainty": {
          "lower": 17.21,
          "upper": 23.39,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-xai-grok-4-6-none-grok-build.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "39.2 min"
          }
        ]
      },
      {
        "id": "2026-09-01-gemini-gemini-3-8-flash-high-mini-swe-agent",
        "label": "Gemini 3.8 Flash · mini-SWE-agent · high",
        "model": "Gemini 3.8 Flash",
        "provider": "Google",
        "harness": "mini-SWE-agent 2.4.6",
        "effort": "high",
        "score": 19.09,
        "costUsd": 0,
        "uncertainty": {
          "lower": 15.73,
          "upper": 22.45,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-09-01-gemini-gemini-3-8-flash-high-mini-swe-agent.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "33.3 min"
          }
        ]
      },
      {
        "id": "2026-08-26-openai-gpt-5-6-luna-max-codex",
        "label": "GPT-5.6 Luna · Codex · max",
        "model": "GPT-5.6 Luna",
        "provider": "OpenAI",
        "harness": "Codex 0.149.1",
        "effort": "max",
        "score": 17.27,
        "costUsd": 346.67,
        "uncertainty": {
          "lower": 14.42,
          "upper": 20.12,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-openai-gpt-5-6-luna-max-codex.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "68.1 min"
          }
        ]
      },
      {
        "id": "2026-08-26-anthropic-claude-sonnet-5-max-claude-code",
        "label": "Sonnet 5 · Claude Code · max",
        "model": "Sonnet 5",
        "provider": "Anthropic",
        "harness": "Claude Code 2.1.231",
        "effort": "max",
        "score": 12.42,
        "costUsd": 9603.86,
        "uncertainty": {
          "lower": 9.36,
          "upper": 15.48,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-anthropic-claude-sonnet-5-max-claude-code.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "108.5 min"
          }
        ]
      },
      {
        "id": "2026-08-26-xai-grok-4-5-none-grok-build",
        "label": "Grok 4.5 · Grok Build · none",
        "model": "Grok 4.5",
        "provider": "xAI",
        "harness": "Grok Build 1.0.5",
        "effort": "none",
        "score": 12.42,
        "costUsd": 2094.11,
        "uncertainty": {
          "lower": 9.8,
          "upper": 15.04,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-08-26-xai-grok-4-5-none-grok-build.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "52.7 min"
          }
        ]
      },
      {
        "id": "2026-09-02-gemini-gemini-3-7-flash-high-mini-swe-agent",
        "label": "Gemini 3.7 Flash · mini-SWE-agent · high",
        "model": "Gemini 3.7 Flash",
        "provider": "Google",
        "harness": "mini-SWE-agent 2.4.6",
        "effort": "high",
        "score": 11.21,
        "costUsd": 1261.87,
        "uncertainty": {
          "lower": 8.760000000000002,
          "upper": 13.66,
          "label": "Source-reported 95% confidence interval"
        },
        "sourceUrl": "https://raw.githubusercontent.com/harbor-framework/terminal-bench/c7878bd7cbd05d1dd21586b83eb3a1929a412312/leaderboard/submissions/2026-09-02-gemini-gemini-3-7-flash-high-mini-swe-agent.json",
        "details": [
          {
            "label": "Trials",
            "value": "330"
          },
          {
            "label": "Average trial duration",
            "value": "28.1 min"
          }
        ]
      }
    ]
  }
}