{
  "schemaVersion": 1,
  "benchmark": {
    "id": "longmemeval-v2-medium",
    "name": "LongMemEval-V2 · Medium",
    "version": "2 · medium · paper baselines",
    "category": "memory",
    "question": "Can an agent reuse what it learned before?",
    "summary": "Compare memory systems that turn previous web-agent work into useful evidence for a later question.",
    "source": {
      "name": "UCLA LongMemEval team",
      "url": "https://xiaowu0162.github.io/longmemeval-v2/",
      "methodologyUrl": "https://github.com/xiaowu0162/LongMemEval-V2"
    },
    "measure": "Answer accuracy with a fixed reader. Query latency is shown for each memory method.",
    "comparisonRule": "Compare memory methods within the same history tier and reader configuration. Small and Medium are separate sets.",
    "limitations": [
      "These are six paper baselines; the public submission leaderboard is still empty.",
      "This measures memory-system plus reader performance, not a general model ranking.",
      "Query latency excludes the wider application experience; no comparable dollar costs are published."
    ],
    "coverage": "charted",
    "tags": [
      "memory",
      "retrieval",
      "agent memory",
      "latency",
      "long context"
    ]
  },
  "dataset": {
    "benchmarkId": "longmemeval-v2-medium",
    "version": "2 · medium · paper baselines",
    "score": {
      "label": "Answer accuracy",
      "unit": "%",
      "direction": "higher",
      "minimum": 0,
      "maximum": 100
    },
    "source": {
      "name": "UCLA LongMemEval-V2 paper baselines",
      "url": "https://xiaowu0162.github.io/longmemeval-v2/",
      "retrievedAt": "2026-09-09T00:19:36.880Z",
      "revision": "Paper baselines; evaluation code 2cc8c540bdb87fe6761629b585e727e1c4704520"
    },
    "evidenceLabel": "Paper baselines · community submissions pending",
    "configurationLabel": "Memory method with a fixed reader",
    "comparabilityNote": "Medium history tier only. This compares six memory methods, not six foundation models. Latency is in the inspector; dollar cost is not published.",
    "points": [
      {
        "id": "agentrunbook-c",
        "label": "AgentRunbook-C · Qwen3.5-9B reader",
        "model": "AgentRunbook-C",
        "provider": "UCLA paper baselines",
        "harness": "Qwen3.5-9B reader",
        "effort": null,
        "score": 70.1,
        "costUsd": null,
        "uncertainty": null,
        "sourceUrl": "https://xiaowu0162.github.io/longmemeval-v2/",
        "details": [
          {
            "label": "Query latency",
            "value": "139.9 seconds"
          },
          {
            "label": "Method family",
            "value": "Coding agent"
          },
          {
            "label": "Reader",
            "value": "Qwen3.5-9B (fixed)"
          },
          {
            "label": "Reference code configuration",
            "value": "Codex 0.117.0 · GPT-5.4-mini · xhigh"
          },
          {
            "label": "Cohort",
            "value": "medium · published paper baselines"
          }
        ]
      },
      {
        "id": "agentrunbook-r",
        "label": "AgentRunbook-R · Qwen3.5-9B reader",
        "model": "AgentRunbook-R",
        "provider": "UCLA paper baselines",
        "harness": "Qwen3.5-9B reader",
        "effort": null,
        "score": 57,
        "costUsd": null,
        "uncertainty": null,
        "sourceUrl": "https://xiaowu0162.github.io/longmemeval-v2/",
        "details": [
          {
            "label": "Query latency",
            "value": "25.8 seconds"
          },
          {
            "label": "Method family",
            "value": "RAG"
          },
          {
            "label": "Reader",
            "value": "Qwen3.5-9B (fixed)"
          },
          {
            "label": "Reference code configuration",
            "value": "See source for memory configuration"
          },
          {
            "label": "Cohort",
            "value": "medium · published paper baselines"
          }
        ]
      },
      {
        "id": "codex",
        "label": "Codex · Qwen3.5-9B reader",
        "model": "Codex",
        "provider": "UCLA paper baselines",
        "harness": "Qwen3.5-9B reader",
        "effort": null,
        "score": 68.7,
        "costUsd": null,
        "uncertainty": null,
        "sourceUrl": "https://xiaowu0162.github.io/longmemeval-v2/",
        "details": [
          {
            "label": "Query latency",
            "value": "185.8 seconds"
          },
          {
            "label": "Method family",
            "value": "Coding agent"
          },
          {
            "label": "Reader",
            "value": "Qwen3.5-9B (fixed)"
          },
          {
            "label": "Reference code configuration",
            "value": "Codex 0.117.0 · GPT-5.4-mini · xhigh"
          },
          {
            "label": "Cohort",
            "value": "medium · published paper baselines"
          }
        ]
      },
      {
        "id": "no-retrieval",
        "label": "No retrieval · Qwen3.5-9B reader",
        "model": "No retrieval",
        "provider": "UCLA paper baselines",
        "harness": "Qwen3.5-9B reader",
        "effort": null,
        "score": 1.3,
        "costUsd": null,
        "uncertainty": null,
        "sourceUrl": "https://xiaowu0162.github.io/longmemeval-v2/",
        "details": [
          {
            "label": "Query latency",
            "value": "0 seconds"
          },
          {
            "label": "Method family",
            "value": "Reader only"
          },
          {
            "label": "Reader",
            "value": "Qwen3.5-9B (fixed)"
          },
          {
            "label": "Reference code configuration",
            "value": "See source for memory configuration"
          },
          {
            "label": "Cohort",
            "value": "medium · published paper baselines"
          }
        ]
      },
      {
        "id": "rag-query-to-slice",
        "label": "RAG: query to slice · Qwen3.5-9B reader",
        "model": "RAG: query to slice",
        "provider": "UCLA paper baselines",
        "harness": "Qwen3.5-9B reader",
        "effort": null,
        "score": 38.1,
        "costUsd": null,
        "uncertainty": null,
        "sourceUrl": "https://xiaowu0162.github.io/longmemeval-v2/",
        "details": [
          {
            "label": "Query latency",
            "value": "0.1 seconds"
          },
          {
            "label": "Method family",
            "value": "RAG"
          },
          {
            "label": "Reader",
            "value": "Qwen3.5-9B (fixed)"
          },
          {
            "label": "Reference code configuration",
            "value": "See source for memory configuration"
          },
          {
            "label": "Cohort",
            "value": "medium · published paper baselines"
          }
        ]
      },
      {
        "id": "rag-query-to-slice-notes",
        "label": "RAG: query to slice + notes · Qwen3.5-9B reader",
        "model": "RAG: query to slice + notes",
        "provider": "UCLA paper baselines",
        "harness": "Qwen3.5-9B reader",
        "effort": null,
        "score": 45.9,
        "costUsd": null,
        "uncertainty": null,
        "sourceUrl": "https://xiaowu0162.github.io/longmemeval-v2/",
        "details": [
          {
            "label": "Query latency",
            "value": "0.3 seconds"
          },
          {
            "label": "Method family",
            "value": "RAG"
          },
          {
            "label": "Reader",
            "value": "Qwen3.5-9B (fixed)"
          },
          {
            "label": "Reference code configuration",
            "value": "See source for memory configuration"
          },
          {
            "label": "Cohort",
            "value": "medium · published paper baselines"
          }
        ]
      }
    ]
  }
}