{ "schemaVersion": 1, "benchmark": { "id": "vals-legal-research", "name": "Legal Research Bench · Vals AI", "version": "1 · private test set", "category": "work", "question": "Can an agent do legal research it can cite?", "summary": "Case and statute research across eight areas of US law, scored against citation-backed answers.", "source": { "name": "Vals AI", "url": "https://www.vals.ai/benchmarks/legal_research", "methodologyUrl": "https://www.vals.ai/about" }, "measure": "Accuracy on research questions across administrative, business, civil, constitutional, criminal, family, health, and immigration law.", "comparisonRule": "Compare within Legal Research Bench version 1. Per-area scores are components, not separate leaderboards.", "limitations": [ "The test set is private and was built with partner law firms, so no independent party can reproduce these scores.", "US law only. The result says nothing about research in another jurisdiction.", "A citation-backed answer that scores well is not legal advice and has not been checked by a lawyer for a specific matter.", "Model names are the identifiers Vals publishes, not aicharts canonical names." ], "coverage": "charted", "tags": [ "legal", "research", "citations", "case law", "agentic", "professional work", "Vals AI", "independent evaluator" ] }, "dataset": { "benchmarkId": "vals-legal-research", "version": "1 · private test set", "score": { "label": "Accuracy", "unit": "%", "direction": "higher", "minimum": 0, "maximum": 100 }, "source": { "name": "Vals AI · Legal Research Bench", "url": "https://www.vals.ai/benchmarks/legal_research", "retrievedAt": "2026-09-28T18:16:23.422Z", "revision": "legal_research v1" }, "observedAt": "2026-09-27T00:00:00.000Z", "evidenceLabel": "Independent evaluator · private test set", "configurationLabel": "Model at the effort Vals selected", "comparabilityNote": "Private evaluation run by Vals with partner law firms. Dollar values are the publisher's average cost per test. Model names are the identifiers Vals publishes, not aicharts canonical names.", "costLabel": "Average cost per test (USD)", "points": [ { "id": "alibaba-qwen3-6-plus", "label": "alibaba/qwen3.6-plus", "model": "alibaba/qwen3.6-plus", "provider": "Alibaba", "harness": null, "effort": null, "score": 14.904, "costUsd": 0.697997, "uncertainty": { "lower": 12.429, "upper": 17.379, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.6-plus" }, { "label": "Standard error", "value": "±2.48 points" }, { "label": "Mean latency", "value": "659 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "19.7%" }, { "label": "Business & Commercial", "value": "16.3%" }, { "label": "Civil Litigation", "value": "14.8%" }, { "label": "Constitutional / Civil Rights", "value": "10.5%" }, { "label": "Criminal", "value": "10.5%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "30.0%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "74.3%" } ] }, { "id": "alibaba-qwen3-7-max", "label": "alibaba/qwen3.7-max", "model": "alibaba/qwen3.7-max", "provider": "Alibaba", "harness": null, "effort": null, "score": 25.481, "costUsd": 1.400701, "uncertainty": { "lower": 22.452, "upper": 28.51, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.7-max" }, { "label": "Standard error", "value": "±3.03 points" }, { "label": "Mean latency", "value": "809 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "36.4%" }, { "label": "Business & Commercial", "value": "20.9%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "23.7%" }, { "label": "Criminal", "value": "21.1%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "40.0%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "78.4%" } ] }, { "id": "alibaba-qwen3-7-plus", "label": "alibaba/qwen3.7-plus", "model": "alibaba/qwen3.7-plus", "provider": "Alibaba", "harness": null, "effort": null, "score": 16.346, "costUsd": 0.295949, "uncertainty": { "lower": 13.776, "upper": 18.916, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.7-plus" }, { "label": "Standard error", "value": "±2.57 points" }, { "label": "Mean latency", "value": "832 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "15.2%" }, { "label": "Business & Commercial", "value": "14.0%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "21.1%" }, { "label": "Criminal", "value": "15.8%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "26.7%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "75.0%" } ] }, { "id": "alibaba-qwen3-8-27b", "label": "alibaba/qwen3.8-27b", "model": "alibaba/qwen3.8-27b", "provider": "Alibaba", "harness": null, "effort": "xhigh", "score": 36.058, "costUsd": 2.159719, "uncertainty": { "lower": 32.721, "upper": 39.395, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.8-27b" }, { "label": "Standard error", "value": "±3.34 points" }, { "label": "Mean latency", "value": "1,601 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "45.5%" }, { "label": "Business & Commercial", "value": "41.9%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "26.3%" }, { "label": "Criminal", "value": "26.3%" }, { "label": "Family", "value": "22.7%" }, { "label": "Health", "value": "63.3%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "82.6%" } ] }, { "id": "alibaba-qwen3-8-max", "label": "alibaba/qwen3.8-max", "model": "alibaba/qwen3.8-max", "provider": "Alibaba", "harness": null, "effort": null, "score": 47.596, "costUsd": 2.486351, "uncertainty": { "lower": 44.125, "upper": 51.06699999999999, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.8-max" }, { "label": "Standard error", "value": "±3.47 points" }, { "label": "Mean latency", "value": "3,678 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "59.1%" }, { "label": "Business & Commercial", "value": "46.5%" }, { "label": "Civil Litigation", "value": "48.1%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "40.4%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "73.3%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "86.1%" } ] }, { "id": "ant-ling-3-0-flash-2607", "label": "ant/ling-3.0-flash-2607", "model": "ant/ling-3.0-flash-2607", "provider": "Ant", "harness": null, "effort": null, "score": 0, "costUsd": 0.036023, "uncertainty": { "lower": 0, "upper": 0, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "ant/ling-3.0-flash-2607" }, { "label": "Standard error", "value": "±0.00 points" }, { "label": "Mean latency", "value": "212 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "0.0%" }, { "label": "Business & Commercial", "value": "0.0%" }, { "label": "Civil Litigation", "value": "0.0%" }, { "label": "Constitutional / Civil Rights", "value": "0.0%" }, { "label": "Criminal", "value": "0.0%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "0.0%" }, { "label": "Immigration", "value": "0.0%" }, { "label": "Weighted Pass", "value": "64.6%" } ] }, { "id": "ant-ling-3-0-flash-af-rc3", "label": "ant/ling-3.0-flash-af-rc3", "model": "ant/ling-3.0-flash-af-rc3", "provider": "Ant", "harness": null, "effort": null, "score": 15.385, "costUsd": 0.028227, "uncertainty": { "lower": 12.876999999999999, "upper": 17.893, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "ant/ling-3.0-flash-af-rc3" }, { "label": "Standard error", "value": "±2.51 points" }, { "label": "Mean latency", "value": "515 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "18.2%" }, { "label": "Business & Commercial", "value": "14.0%" }, { "label": "Civil Litigation", "value": "22.2%" }, { "label": "Constitutional / Civil Rights", "value": "10.5%" }, { "label": "Criminal", "value": "12.3%" }, { "label": "Family", "value": "4.5%" }, { "label": "Health", "value": "26.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "73.2%" } ] }, { "id": "anthropic-claude-fable-5", "label": "anthropic/claude-fable-5", "model": "anthropic/claude-fable-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 49.519, "costUsd": 9.791426, "uncertainty": { "lower": 46.044, "upper": 52.994, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-fable-5" }, { "label": "Standard error", "value": "±3.48 points" }, { "label": "Mean latency", "value": "1,338 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "57.6%" }, { "label": "Business & Commercial", "value": "51.2%" }, { "label": "Civil Litigation", "value": "44.4%" }, { "label": "Constitutional / Civil Rights", "value": "50.0%" }, { "label": "Criminal", "value": "42.1%" }, { "label": "Family", "value": "36.4%" }, { "label": "Health", "value": "66.7%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "87.8%" } ] }, { "id": "anthropic-claude-fable-5-1", "label": "anthropic/claude-fable-5-1", "model": "anthropic/claude-fable-5-1", "provider": "Anthropic", "harness": null, "effort": "max", "score": 55.288, "costUsd": 23.055789, "uncertainty": { "lower": 51.831999999999994, "upper": 58.744, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-fable-5-1" }, { "label": "Standard error", "value": "±3.46 points" }, { "label": "Mean latency", "value": "3,449 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "68.2%" }, { "label": "Business & Commercial", "value": "62.8%" }, { "label": "Civil Litigation", "value": "63.0%" }, { "label": "Constitutional / Civil Rights", "value": "42.1%" }, { "label": "Criminal", "value": "38.6%" }, { "label": "Family", "value": "31.8%" }, { "label": "Health", "value": "80.0%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "90.2%" } ] }, { "id": "anthropic-claude-haiku-4-5-20251001-thinking", "label": "anthropic/claude-haiku-4-5-20251001-thinking", "model": "anthropic/claude-haiku-4-5-20251001-thinking", "provider": "Anthropic", "harness": null, "effort": null, "score": 10.577, "costUsd": 0.26891, "uncertainty": { "lower": 8.439, "upper": 12.715, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-haiku-4-5-20251001-thinking" }, { "label": "Standard error", "value": "±2.14 points" }, { "label": "Mean latency", "value": "243 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "10.6%" }, { "label": "Business & Commercial", "value": "9.3%" }, { "label": "Civil Litigation", "value": "7.4%" }, { "label": "Constitutional / Civil Rights", "value": "13.2%" }, { "label": "Criminal", "value": "10.5%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "6.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "64.9%" } ] }, { "id": "anthropic-claude-opus-4-7", "label": "anthropic/claude-opus-4-7", "model": "anthropic/claude-opus-4-7", "provider": "Anthropic", "harness": null, "effort": "max", "score": 38.462, "costUsd": 3.05609, "uncertainty": { "lower": 35.081, "upper": 41.843, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-4-7" }, { "label": "Standard error", "value": "±3.38 points" }, { "label": "Mean latency", "value": "512 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "45.5%" }, { "label": "Business & Commercial", "value": "32.6%" }, { "label": "Civil Litigation", "value": "37.0%" }, { "label": "Constitutional / Civil Rights", "value": "44.7%" }, { "label": "Criminal", "value": "29.8%" }, { "label": "Family", "value": "31.8%" }, { "label": "Health", "value": "53.3%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "84.1%" } ] }, { "id": "anthropic-claude-opus-4-8", "label": "anthropic/claude-opus-4-8", "model": "anthropic/claude-opus-4-8", "provider": "Anthropic", "harness": null, "effort": "max", "score": 43.75, "costUsd": 2.820942, "uncertainty": { "lower": 40.302, "upper": 47.198, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-4-8" }, { "label": "Standard error", "value": "±3.45 points" }, { "label": "Mean latency", "value": "714 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "43.9%" }, { "label": "Business & Commercial", "value": "41.9%" }, { "label": "Civil Litigation", "value": "51.9%" }, { "label": "Constitutional / Civil Rights", "value": "39.5%" }, { "label": "Criminal", "value": "45.6%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "53.3%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "86.1%" } ] }, { "id": "anthropic-claude-opus-5", "label": "anthropic/claude-opus-5", "model": "anthropic/claude-opus-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 55.288, "costUsd": 6.762498, "uncertainty": { "lower": 51.831999999999994, "upper": 58.744, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-5" }, { "label": "Standard error", "value": "±3.46 points" }, { "label": "Mean latency", "value": "1,861 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "65.2%" }, { "label": "Business & Commercial", "value": "62.8%" }, { "label": "Civil Litigation", "value": "63.0%" }, { "label": "Constitutional / Civil Rights", "value": "44.7%" }, { "label": "Criminal", "value": "43.9%" }, { "label": "Family", "value": "40.9%" }, { "label": "Health", "value": "76.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "90.6%" } ] }, { "id": "anthropic-claude-opus-5-5", "label": "anthropic/claude-opus-5-5", "model": "anthropic/claude-opus-5-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 50.481, "costUsd": 25.163725, "uncertainty": { "lower": 47.006, "upper": 53.956, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-5-5" }, { "label": "Standard error", "value": "±3.48 points" }, { "label": "Mean latency", "value": "6,750 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "65.2%" }, { "label": "Business & Commercial", "value": "48.8%" }, { "label": "Civil Litigation", "value": "48.1%" }, { "label": "Constitutional / Civil Rights", "value": "42.1%" }, { "label": "Criminal", "value": "42.1%" }, { "label": "Family", "value": "22.7%" }, { "label": "Health", "value": "80.0%" }, { "label": "Immigration", "value": "54.5%" }, { "label": "Weighted Pass", "value": "80.4%" } ] }, { "id": "anthropic-claude-sonnet-4-6", "label": "anthropic/claude-sonnet-4-6", "model": "anthropic/claude-sonnet-4-6", "provider": "Anthropic", "harness": null, "effort": "max", "score": 38.462, "costUsd": 2.302948, "uncertainty": { "lower": 35.081, "upper": 41.843, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-sonnet-4-6" }, { "label": "Standard error", "value": "±3.38 points" }, { "label": "Mean latency", "value": "990 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "43.9%" }, { "label": "Business & Commercial", "value": "34.9%" }, { "label": "Civil Litigation", "value": "29.6%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "36.8%" }, { "label": "Family", "value": "27.3%" }, { "label": "Health", "value": "53.3%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "85.3%" } ] }, { "id": "anthropic-claude-sonnet-5", "label": "anthropic/claude-sonnet-5", "model": "anthropic/claude-sonnet-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 41.827, "costUsd": 2.719446, "uncertainty": { "lower": 38.399, "upper": 45.254999999999995, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-sonnet-5" }, { "label": "Standard error", "value": "±3.43 points" }, { "label": "Mean latency", "value": "1,546 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "54.5%" }, { "label": "Business & Commercial", "value": "32.6%" }, { "label": "Civil Litigation", "value": "44.4%" }, { "label": "Constitutional / Civil Rights", "value": "44.7%" }, { "label": "Criminal", "value": "40.4%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "63.3%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "84.6%" } ] }, { "id": "anthropic-claude-sonnet-5-5", "label": "anthropic/claude-sonnet-5-5", "model": "anthropic/claude-sonnet-5-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 48.077, "costUsd": 14.826571, "uncertainty": { "lower": 44.604, "upper": 51.55, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-sonnet-5-5" }, { "label": "Standard error", "value": "±3.47 points" }, { "label": "Mean latency", "value": "5,589 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "69.7%" }, { "label": "Business & Commercial", "value": "39.5%" }, { "label": "Civil Litigation", "value": "48.1%" }, { "label": "Constitutional / Civil Rights", "value": "42.1%" }, { "label": "Criminal", "value": "38.6%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "80.0%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "84.3%" } ] }, { "id": "deepseek-deepseek-v4-1-flash", "label": "deepseek/deepseek-v4.1-flash", "model": "deepseek/deepseek-v4.1-flash", "provider": "DeepSeek", "harness": null, "effort": "high", "score": 41.346, "costUsd": 0.24793, "uncertainty": { "lower": 37.922999999999995, "upper": 44.769, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4.1-flash" }, { "label": "Standard error", "value": "±3.42 points" }, { "label": "Mean latency", "value": "702 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "53.0%" }, { "label": "Business & Commercial", "value": "39.5%" }, { "label": "Civil Litigation", "value": "48.1%" }, { "label": "Constitutional / Civil Rights", "value": "36.8%" }, { "label": "Criminal", "value": "33.3%" }, { "label": "Family", "value": "13.6%" }, { "label": "Health", "value": "73.3%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "84.5%" } ] }, { "id": "deepseek-deepseek-v4-flash-0731", "label": "deepseek/deepseek-v4-flash-0731", "model": "deepseek/deepseek-v4-flash-0731", "provider": "DeepSeek", "harness": null, "effort": "high", "score": 30.288, "costUsd": 0.275172, "uncertainty": { "lower": 27.094, "upper": 33.482, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4-flash-0731" }, { "label": "Standard error", "value": "±3.19 points" }, { "label": "Mean latency", "value": "1,267 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "36.4%" }, { "label": "Business & Commercial", "value": "23.3%" }, { "label": "Civil Litigation", "value": "44.4%" }, { "label": "Constitutional / Civil Rights", "value": "31.6%" }, { "label": "Criminal", "value": "24.6%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "36.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "81.2%" } ] }, { "id": "deepseek-deepseek-v4-pro", "label": "deepseek/deepseek-v4-pro", "model": "deepseek/deepseek-v4-pro", "provider": "DeepSeek", "harness": null, "effort": "max", "score": 23.077, "costUsd": 0.682218, "uncertainty": { "lower": 20.149, "upper": 26.005000000000003, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4-pro" }, { "label": "Standard error", "value": "±2.93 points" }, { "label": "Mean latency", "value": "1,025 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "28.8%" }, { "label": "Business & Commercial", "value": "23.3%" }, { "label": "Civil Litigation", "value": "33.3%" }, { "label": "Constitutional / Civil Rights", "value": "13.2%" }, { "label": "Criminal", "value": "15.8%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "46.7%" }, { "label": "Immigration", "value": "9.1%" }, { "label": "Weighted Pass", "value": "80.7%" } ] }, { "id": "deepseek-deepseek-v4-pro-0813", "label": "deepseek/deepseek-v4-pro-0813", "model": "deepseek/deepseek-v4-pro-0813", "provider": "DeepSeek", "harness": null, "effort": "max", "score": 40.865, "costUsd": 1.088017, "uncertainty": { "lower": 37.448, "upper": 44.282000000000004, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4-pro-0813" }, { "label": "Standard error", "value": "±3.42 points" }, { "label": "Mean latency", "value": "2,527 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "50.0%" }, { "label": "Business & Commercial", "value": "34.9%" }, { "label": "Civil Litigation", "value": "40.7%" }, { "label": "Constitutional / Civil Rights", "value": "39.5%" }, { "label": "Criminal", "value": "35.1%" }, { "label": "Family", "value": "22.7%" }, { "label": "Health", "value": "60.0%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "83.8%" } ] }, { "id": "fireworks-nemotron-lightning-3p5-30b-a3b", "label": "fireworks/nemotron-lightning-3p5-30b-a3b", "model": "fireworks/nemotron-lightning-3p5-30b-a3b", "provider": "Fireworks AI", "harness": null, "effort": null, "score": 2.885, "costUsd": 0.004365, "uncertainty": { "lower": 1.7219999999999998, "upper": 4.048, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "fireworks/nemotron-lightning-3p5-30b-a3b" }, { "label": "Standard error", "value": "±1.16 points" }, { "label": "Mean latency", "value": "285 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "1.5%" }, { "label": "Business & Commercial", "value": "4.7%" }, { "label": "Civil Litigation", "value": "3.7%" }, { "label": "Constitutional / Civil Rights", "value": "5.3%" }, { "label": "Criminal", "value": "3.5%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "0.0%" }, { "label": "Immigration", "value": "0.0%" }, { "label": "Weighted Pass", "value": "49.0%" } ] }, { "id": "google-gemini-3-1-flash-lite-preview", "label": "google/gemini-3.1-flash-lite-preview", "model": "google/gemini-3.1-flash-lite-preview", "provider": "Google", "harness": null, "effort": null, "score": 3.365, "costUsd": 0.013607, "uncertainty": { "lower": 2.112, "upper": 4.618, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.1-flash-lite-preview" }, { "label": "Standard error", "value": "±1.25 points" }, { "label": "Mean latency", "value": "20 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "3.0%" }, { "label": "Business & Commercial", "value": "2.3%" }, { "label": "Civil Litigation", "value": "3.7%" }, { "label": "Constitutional / Civil Rights", "value": "2.6%" }, { "label": "Criminal", "value": "1.8%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "6.7%" }, { "label": "Immigration", "value": "9.1%" }, { "label": "Weighted Pass", "value": "54.4%" } ] }, { "id": "google-gemini-3-1-pro-preview", "label": "google/gemini-3.1-pro-preview", "model": "google/gemini-3.1-pro-preview", "provider": "Google", "harness": null, "effort": "high", "score": 20.673, "costUsd": 1.100145, "uncertainty": { "lower": 17.857999999999997, "upper": 23.488, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.1-pro-preview" }, { "label": "Standard error", "value": "±2.81 points" }, { "label": "Mean latency", "value": "242 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "24.2%" }, { "label": "Business & Commercial", "value": "23.3%" }, { "label": "Civil Litigation", "value": "11.1%" }, { "label": "Constitutional / Civil Rights", "value": "21.1%" }, { "label": "Criminal", "value": "19.3%" }, { "label": "Family", "value": "4.5%" }, { "label": "Health", "value": "33.3%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "76.6%" } ] }, { "id": "google-gemini-3-5-flash", "label": "google/gemini-3.5-flash", "model": "google/gemini-3.5-flash", "provider": "Google", "harness": null, "effort": "high", "score": 30.769, "costUsd": 1.305281, "uncertainty": { "lower": 27.561, "upper": 33.977, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.5-flash" }, { "label": "Standard error", "value": "±3.21 points" }, { "label": "Mean latency", "value": "278 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "43.9%" }, { "label": "Business & Commercial", "value": "27.9%" }, { "label": "Civil Litigation", "value": "22.2%" }, { "label": "Constitutional / Civil Rights", "value": "28.9%" }, { "label": "Criminal", "value": "26.3%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "43.3%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "83.9%" } ] }, { "id": "google-gemini-3-5-flash-lite", "label": "google/gemini-3.5-flash-lite", "model": "google/gemini-3.5-flash-lite", "provider": "Google", "harness": null, "effort": "high", "score": 13.942, "costUsd": 0.124723, "uncertainty": { "lower": 11.534, "upper": 16.35, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.5-flash-lite" }, { "label": "Standard error", "value": "±2.41 points" }, { "label": "Mean latency", "value": "59 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "19.7%" }, { "label": "Business & Commercial", "value": "7.0%" }, { "label": "Civil Litigation", "value": "11.1%" }, { "label": "Constitutional / Civil Rights", "value": "21.1%" }, { "label": "Criminal", "value": "12.3%" }, { "label": "Family", "value": "4.5%" }, { "label": "Health", "value": "13.3%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "68.9%" } ] }, { "id": "google-gemini-3-6-flash", "label": "google/gemini-3.6-flash", "model": "google/gemini-3.6-flash", "provider": "Google", "harness": null, "effort": "high", "score": 25, "costUsd": 0.466741, "uncertainty": { "lower": 21.990000000000002, "upper": 28.009999999999998, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.6-flash" }, { "label": "Standard error", "value": "±3.01 points" }, { "label": "Mean latency", "value": "168 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "31.8%" }, { "label": "Business & Commercial", "value": "23.3%" }, { "label": "Civil Litigation", "value": "18.5%" }, { "label": "Constitutional / Civil Rights", "value": "18.4%" }, { "label": "Criminal", "value": "24.6%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "36.7%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "80.5%" } ] }, { "id": "google-gemini-3-7-flash", "label": "google/gemini-3.7-flash", "model": "google/gemini-3.7-flash", "provider": "Google", "harness": null, "effort": "high", "score": 34.615, "costUsd": 0.760397, "uncertainty": { "lower": 31.308000000000003, "upper": 37.922000000000004, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.7-flash" }, { "label": "Standard error", "value": "±3.31 points" }, { "label": "Mean latency", "value": "289 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "42.4%" }, { "label": "Business & Commercial", "value": "37.2%" }, { "label": "Civil Litigation", "value": "29.6%" }, { "label": "Constitutional / Civil Rights", "value": "31.6%" }, { "label": "Criminal", "value": "33.3%" }, { "label": "Family", "value": "13.6%" }, { "label": "Health", "value": "50.0%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "84.1%" } ] }, { "id": "google-gemini-3-8-flash", "label": "google/gemini-3.8-flash", "model": "google/gemini-3.8-flash", "provider": "Google", "harness": null, "effort": "high", "score": 38.942, "costUsd": 1.630498, "uncertainty": { "lower": 35.553, "upper": 42.331, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.8-flash" }, { "label": "Standard error", "value": "±3.39 points" }, { "label": "Mean latency", "value": "324 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "47.0%" }, { "label": "Business & Commercial", "value": "41.9%" }, { "label": "Civil Litigation", "value": "29.6%" }, { "label": "Constitutional / Civil Rights", "value": "42.1%" }, { "label": "Criminal", "value": "40.4%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "56.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "87.5%" } ] }, { "id": "google-gemini-3-flash-preview", "label": "google/gemini-3-flash-preview", "model": "google/gemini-3-flash-preview", "provider": "Google", "harness": null, "effort": "high", "score": 18.269, "costUsd": 0.135395, "uncertainty": { "lower": 15.582999999999998, "upper": 20.955, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "google/gemini-3-flash-preview" }, { "label": "Standard error", "value": "±2.69 points" }, { "label": "Mean latency", "value": "188 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "27.3%" }, { "label": "Business & Commercial", "value": "18.6%" }, { "label": "Civil Litigation", "value": "11.1%" }, { "label": "Constitutional / Civil Rights", "value": "7.9%" }, { "label": "Criminal", "value": "12.3%" }, { "label": "Family", "value": "4.5%" }, { "label": "Health", "value": "30.0%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "75.2%" } ] }, { "id": "grok-grok-4-20-0309-reasoning", "label": "grok/grok-4.20-0309-reasoning", "model": "grok/grok-4.20-0309-reasoning", "provider": "SpaceXAI", "harness": null, "effort": null, "score": 13.942, "costUsd": 0.261499, "uncertainty": { "lower": 11.534, "upper": 16.35, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.20-0309-reasoning" }, { "label": "Standard error", "value": "±2.41 points" }, { "label": "Mean latency", "value": "244 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "9.1%" }, { "label": "Business & Commercial", "value": "16.3%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "18.4%" }, { "label": "Criminal", "value": "17.5%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "10.0%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "65.1%" } ] }, { "id": "grok-grok-4-3", "label": "grok/grok-4.3", "model": "grok/grok-4.3", "provider": "SpaceXAI", "harness": null, "effort": "high", "score": 15.385, "costUsd": 0.203143, "uncertainty": { "lower": 12.876999999999999, "upper": 17.893, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.3" }, { "label": "Standard error", "value": "±2.51 points" }, { "label": "Mean latency", "value": "85 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "25.8%" }, { "label": "Business & Commercial", "value": "14.0%" }, { "label": "Civil Litigation", "value": "3.7%" }, { "label": "Constitutional / Civil Rights", "value": "13.2%" }, { "label": "Criminal", "value": "14.0%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "40.0%" }, { "label": "Immigration", "value": "9.1%" }, { "label": "Weighted Pass", "value": "69.5%" } ] }, { "id": "grok-grok-4-5", "label": "grok/grok-4.5", "model": "grok/grok-4.5", "provider": "SpaceXAI", "harness": null, "effort": "high", "score": 37.981, "costUsd": 1.086814, "uncertainty": { "lower": 34.608000000000004, "upper": 41.354, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.5" }, { "label": "Standard error", "value": "±3.37 points" }, { "label": "Mean latency", "value": "954 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "51.5%" }, { "label": "Business & Commercial", "value": "30.2%" }, { "label": "Civil Litigation", "value": "33.3%" }, { "label": "Constitutional / Civil Rights", "value": "31.6%" }, { "label": "Criminal", "value": "36.8%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "66.7%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "84.4%" } ] }, { "id": "grok-grok-4-6", "label": "grok/grok-4.6", "model": "grok/grok-4.6", "provider": "SpaceXAI", "harness": null, "effort": "high", "score": 48.077, "costUsd": 1.530204, "uncertainty": { "lower": 44.604, "upper": 51.55, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.6" }, { "label": "Standard error", "value": "±3.47 points" }, { "label": "Mean latency", "value": "1,504 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "62.1%" }, { "label": "Business & Commercial", "value": "48.8%" }, { "label": "Civil Litigation", "value": "37.0%" }, { "label": "Constitutional / Civil Rights", "value": "36.8%" }, { "label": "Criminal", "value": "42.1%" }, { "label": "Family", "value": "22.7%" }, { "label": "Health", "value": "73.3%" }, { "label": "Immigration", "value": "54.5%" }, { "label": "Weighted Pass", "value": "88.3%" } ] }, { "id": "grok-grok-4-7", "label": "grok/grok-4.7", "model": "grok/grok-4.7", "provider": "SpaceXAI", "harness": null, "effort": "xhigh", "score": 47.115, "costUsd": 4.920812, "uncertainty": { "lower": 43.646, "upper": 50.584, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.7" }, { "label": "Standard error", "value": "±3.47 points" }, { "label": "Mean latency", "value": "1,274 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "54.5%" }, { "label": "Business & Commercial", "value": "44.2%" }, { "label": "Civil Litigation", "value": "44.4%" }, { "label": "Constitutional / Civil Rights", "value": "42.1%" }, { "label": "Criminal", "value": "47.4%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "76.7%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "86.6%" } ] }, { "id": "inception-mercury-2-5", "label": "inception/mercury-2.5", "model": "inception/mercury-2.5", "provider": "Inception", "harness": null, "effort": "high", "score": 4.327, "costUsd": 0.058112, "uncertainty": { "lower": 2.9130000000000003, "upper": 5.741, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "inception/mercury-2.5" }, { "label": "Standard error", "value": "±1.41 points" }, { "label": "Mean latency", "value": "46 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "4.5%" }, { "label": "Business & Commercial", "value": "11.6%" }, { "label": "Civil Litigation", "value": "3.7%" }, { "label": "Constitutional / Civil Rights", "value": "2.6%" }, { "label": "Criminal", "value": "1.8%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "3.3%" }, { "label": "Immigration", "value": "0.0%" }, { "label": "Weighted Pass", "value": "58.9%" } ] }, { "id": "kimi-kimi-k2-5-thinking", "label": "kimi/kimi-k2.5-thinking", "model": "kimi/kimi-k2.5-thinking", "provider": "Moonshot AI", "harness": null, "effort": null, "score": 15.865, "costUsd": 0.23035, "uncertainty": { "lower": 13.326, "upper": 18.404, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "kimi/kimi-k2.5-thinking" }, { "label": "Standard error", "value": "±2.54 points" }, { "label": "Mean latency", "value": "1,333 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "16.7%" }, { "label": "Business & Commercial", "value": "14.0%" }, { "label": "Civil Litigation", "value": "18.5%" }, { "label": "Constitutional / Civil Rights", "value": "15.8%" }, { "label": "Criminal", "value": "15.8%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "10.0%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "71.3%" } ] }, { "id": "kimi-kimi-k2-6", "label": "kimi/kimi-k2.6", "model": "kimi/kimi-k2.6", "provider": "Moonshot AI", "harness": null, "effort": null, "score": 15.865, "costUsd": 1.520305, "uncertainty": { "lower": 13.326, "upper": 18.404, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "kimi/kimi-k2.6" }, { "label": "Standard error", "value": "±2.54 points" }, { "label": "Mean latency", "value": "1,853 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "18.2%" }, { "label": "Business & Commercial", "value": "20.9%" }, { "label": "Civil Litigation", "value": "18.5%" }, { "label": "Constitutional / Civil Rights", "value": "21.1%" }, { "label": "Criminal", "value": "14.0%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "30.0%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "74.7%" } ] }, { "id": "kimi-kimi-k3", "label": "kimi/kimi-k3", "model": "kimi/kimi-k3", "provider": "Moonshot AI", "harness": null, "effort": null, "score": 44.231, "costUsd": 3.222748, "uncertainty": { "lower": 40.779, "upper": 47.683, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "kimi/kimi-k3" }, { "label": "Standard error", "value": "±3.45 points" }, { "label": "Mean latency", "value": "2,146 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "54.5%" }, { "label": "Business & Commercial", "value": "46.5%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "44.7%" }, { "label": "Criminal", "value": "36.8%" }, { "label": "Family", "value": "27.3%" }, { "label": "Health", "value": "63.3%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "87.0%" } ] }, { "id": "meta-muse-spark-1-1", "label": "meta/muse_spark_1_1", "model": "meta/muse_spark_1_1", "provider": "Meta", "harness": null, "effort": "xhigh", "score": 37.981, "costUsd": 0.377379, "uncertainty": { "lower": 34.608000000000004, "upper": 41.354, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_1" }, { "label": "Standard error", "value": "±3.37 points" }, { "label": "Mean latency", "value": "313 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "37.9%" }, { "label": "Business & Commercial", "value": "51.2%" }, { "label": "Civil Litigation", "value": "48.1%" }, { "label": "Constitutional / Civil Rights", "value": "28.9%" }, { "label": "Criminal", "value": "36.8%" }, { "label": "Family", "value": "22.7%" }, { "label": "Health", "value": "40.0%" }, { "label": "Immigration", "value": "9.1%" }, { "label": "Weighted Pass", "value": "84.6%" } ] }, { "id": "meta-muse-spark-1-2", "label": "meta/muse_spark_1_2", "model": "meta/muse_spark_1_2", "provider": "Meta", "harness": null, "effort": "xhigh", "score": 43.75, "costUsd": 0.508097, "uncertainty": { "lower": 40.302, "upper": 47.198, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_2" }, { "label": "Standard error", "value": "±3.45 points" }, { "label": "Mean latency", "value": "307 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "48.5%" }, { "label": "Business & Commercial", "value": "44.2%" }, { "label": "Civil Litigation", "value": "37.0%" }, { "label": "Constitutional / Civil Rights", "value": "39.5%" }, { "label": "Criminal", "value": "38.6%" }, { "label": "Family", "value": "27.3%" }, { "label": "Health", "value": "56.7%" }, { "label": "Immigration", "value": "54.5%" }, { "label": "Weighted Pass", "value": "85.3%" } ] }, { "id": "meta-muse-spark-1-3", "label": "meta/muse_spark_1_3", "model": "meta/muse_spark_1_3", "provider": "Meta", "harness": null, "effort": "xhigh", "score": 40.865, "costUsd": 0.650063, "uncertainty": { "lower": 37.448, "upper": 44.282000000000004, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_3" }, { "label": "Standard error", "value": "±3.42 points" }, { "label": "Mean latency", "value": "531 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "51.5%" }, { "label": "Business & Commercial", "value": "32.6%" }, { "label": "Civil Litigation", "value": "40.7%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "35.1%" }, { "label": "Family", "value": "27.3%" }, { "label": "Health", "value": "70.0%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "85.2%" } ] }, { "id": "meta-muse-spark-1-3-max", "label": "meta/muse_spark_1_3_max", "model": "meta/muse_spark_1_3_max", "provider": "Meta", "harness": null, "effort": "max", "score": 55.288, "costUsd": 0.588812, "uncertainty": { "lower": 51.831999999999994, "upper": 58.744, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_3_max" }, { "label": "Standard error", "value": "±3.46 points" }, { "label": "Mean latency", "value": "317 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "60.6%" }, { "label": "Business & Commercial", "value": "58.1%" }, { "label": "Civil Litigation", "value": "63.0%" }, { "label": "Constitutional / Civil Rights", "value": "50.0%" }, { "label": "Criminal", "value": "47.4%" }, { "label": "Family", "value": "40.9%" }, { "label": "Health", "value": "73.3%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "86.8%" } ] }, { "id": "minimax-minimax-m2-7", "label": "minimax/MiniMax-M2.7", "model": "minimax/MiniMax-M2.7", "provider": "MiniMax", "harness": null, "effort": null, "score": 10.577, "costUsd": 0.219167, "uncertainty": { "lower": 8.439, "upper": 12.715, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "minimax/MiniMax-M2.7" }, { "label": "Standard error", "value": "±2.14 points" }, { "label": "Mean latency", "value": "421 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "16.7%" }, { "label": "Business & Commercial", "value": "14.0%" }, { "label": "Civil Litigation", "value": "14.8%" }, { "label": "Constitutional / Civil Rights", "value": "10.5%" }, { "label": "Criminal", "value": "5.3%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "13.3%" }, { "label": "Immigration", "value": "0.0%" }, { "label": "Weighted Pass", "value": "63.9%" } ] }, { "id": "minimax-minimax-m3", "label": "minimax/MiniMax-M3", "model": "minimax/MiniMax-M3", "provider": "MiniMax", "harness": null, "effort": null, "score": 29.808, "costUsd": 0.342955, "uncertainty": { "lower": 26.629, "upper": 32.987, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "minimax/MiniMax-M3" }, { "label": "Standard error", "value": "±3.18 points" }, { "label": "Mean latency", "value": "814 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "36.4%" }, { "label": "Business & Commercial", "value": "25.6%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "26.3%" }, { "label": "Family", "value": "13.6%" }, { "label": "Health", "value": "46.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "81.1%" } ] }, { "id": "mistralai-mistral-medium-3-5", "label": "mistralai/mistral-medium-3.5", "model": "mistralai/mistral-medium-3.5", "provider": "Mistral AI", "harness": null, "effort": "high", "score": 9.135, "costUsd": 1.308079, "uncertainty": { "lower": 7.133, "upper": 11.137, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "mistralai/mistral-medium-3.5" }, { "label": "Standard error", "value": "±2.00 points" }, { "label": "Mean latency", "value": "613 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "7.6%" }, { "label": "Business & Commercial", "value": "9.3%" }, { "label": "Civil Litigation", "value": "11.1%" }, { "label": "Constitutional / Civil Rights", "value": "10.5%" }, { "label": "Criminal", "value": "10.5%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "6.7%" }, { "label": "Immigration", "value": "9.1%" }, { "label": "Weighted Pass", "value": "63.3%" } ] }, { "id": "nvidia-nemotron-3-ultra-550b-a55b", "label": "nvidia/nemotron-3-ultra-550b-a55b", "model": "nvidia/nemotron-3-ultra-550b-a55b", "provider": "Nvidia", "harness": null, "effort": null, "score": 15.385, "costUsd": null, "uncertainty": { "lower": 12.876999999999999, "upper": 17.893, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "nvidia/nemotron-3-ultra-550b-a55b" }, { "label": "Standard error", "value": "±2.51 points" }, { "label": "Mean latency", "value": "611 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "16.7%" }, { "label": "Business & Commercial", "value": "11.6%" }, { "label": "Civil Litigation", "value": "22.2%" }, { "label": "Constitutional / Civil Rights", "value": "13.2%" }, { "label": "Criminal", "value": "15.8%" }, { "label": "Family", "value": "4.5%" }, { "label": "Health", "value": "16.7%" }, { "label": "Immigration", "value": "9.1%" }, { "label": "Weighted Pass", "value": "68.0%" } ] }, { "id": "openai-gpt-5-4-mini-2026-03-17", "label": "openai/gpt-5.4-mini-2026-03-17", "model": "openai/gpt-5.4-mini-2026-03-17", "provider": "OpenAI", "harness": null, "effort": "xhigh", "score": 12.5, "costUsd": 1.864673, "uncertainty": { "lower": 10.201, "upper": 14.799, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.4-mini-2026-03-17" }, { "label": "Standard error", "value": "±2.30 points" }, { "label": "Mean latency", "value": "2,256 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "15.2%" }, { "label": "Business & Commercial", "value": "11.6%" }, { "label": "Civil Litigation", "value": "14.8%" }, { "label": "Constitutional / Civil Rights", "value": "13.2%" }, { "label": "Criminal", "value": "5.3%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "16.7%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "67.3%" } ] }, { "id": "openai-gpt-5-4-nano-2026-03-17", "label": "openai/gpt-5.4-nano-2026-03-17", "model": "openai/gpt-5.4-nano-2026-03-17", "provider": "OpenAI", "harness": null, "effort": "high", "score": 6.25, "costUsd": 0.141253, "uncertainty": { "lower": 4.568, "upper": 7.932, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.4-nano-2026-03-17" }, { "label": "Standard error", "value": "±1.68 points" }, { "label": "Mean latency", "value": "527 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "10.6%" }, { "label": "Business & Commercial", "value": "7.0%" }, { "label": "Civil Litigation", "value": "7.4%" }, { "label": "Constitutional / Civil Rights", "value": "2.6%" }, { "label": "Criminal", "value": "3.5%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "13.3%" }, { "label": "Immigration", "value": "0.0%" }, { "label": "Weighted Pass", "value": "62.0%" } ] }, { "id": "openai-gpt-5-5", "label": "openai/gpt-5.5", "model": "openai/gpt-5.5", "provider": "OpenAI", "harness": null, "effort": "xhigh", "score": 40.385, "costUsd": 7.402471, "uncertainty": { "lower": 36.974999999999994, "upper": 43.795, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.5" }, { "label": "Standard error", "value": "±3.41 points" }, { "label": "Mean latency", "value": "2,265 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "53.0%" }, { "label": "Business & Commercial", "value": "37.2%" }, { "label": "Civil Litigation", "value": "44.4%" }, { "label": "Constitutional / Civil Rights", "value": "28.9%" }, { "label": "Criminal", "value": "36.8%" }, { "label": "Family", "value": "13.6%" }, { "label": "Health", "value": "66.7%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "84.0%" } ] }, { "id": "openai-gpt-5-6-luna", "label": "openai/gpt-5.6-luna", "model": "openai/gpt-5.6-luna", "provider": "OpenAI", "harness": null, "effort": "max", "score": 36.538, "costUsd": 0.845008, "uncertainty": { "lower": 33.190999999999995, "upper": 39.885, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.6-luna" }, { "label": "Standard error", "value": "±3.35 points" }, { "label": "Mean latency", "value": "2,374 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "48.5%" }, { "label": "Business & Commercial", "value": "32.6%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "29.8%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "70.0%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "85.0%" } ] }, { "id": "openai-gpt-5-6-sol", "label": "openai/gpt-5.6-sol", "model": "openai/gpt-5.6-sol", "provider": "OpenAI", "harness": null, "effort": "max", "score": 48.077, "costUsd": 21.605566, "uncertainty": { "lower": 44.604, "upper": 51.55, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.6-sol" }, { "label": "Standard error", "value": "±3.47 points" }, { "label": "Mean latency", "value": "4,631 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "60.6%" }, { "label": "Business & Commercial", "value": "53.5%" }, { "label": "Civil Litigation", "value": "40.7%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "43.9%" }, { "label": "Family", "value": "13.6%" }, { "label": "Health", "value": "80.0%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "87.8%" } ] }, { "id": "openai-gpt-5-6-terra", "label": "openai/gpt-5.6-terra", "model": "openai/gpt-5.6-terra", "provider": "OpenAI", "harness": null, "effort": "max", "score": 41.346, "costUsd": 7.907299, "uncertainty": { "lower": 37.922999999999995, "upper": 44.769, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.6-terra" }, { "label": "Standard error", "value": "±3.42 points" }, { "label": "Mean latency", "value": "4,173 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "57.6%" }, { "label": "Business & Commercial", "value": "37.2%" }, { "label": "Civil Litigation", "value": "29.6%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "35.1%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "80.0%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "85.1%" } ] }, { "id": "openai-gpt-6-astra", "label": "openai/gpt-6-astra", "model": "openai/gpt-6-astra", "provider": "OpenAI", "harness": null, "effort": "max", "score": 39.423, "costUsd": 10.577143, "uncertainty": { "lower": 36.026, "upper": 42.82, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-6-astra" }, { "label": "Standard error", "value": "±3.40 points" }, { "label": "Mean latency", "value": "1,491 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "53.0%" }, { "label": "Business & Commercial", "value": "32.6%" }, { "label": "Civil Litigation", "value": "40.7%" }, { "label": "Constitutional / Civil Rights", "value": "26.3%" }, { "label": "Criminal", "value": "29.8%" }, { "label": "Family", "value": "18.2%" }, { "label": "Health", "value": "76.7%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "84.8%" } ] }, { "id": "openai-gpt-6-luna", "label": "openai/gpt-6-luna", "model": "openai/gpt-6-luna", "provider": "OpenAI", "harness": null, "effort": "max", "score": 30.288, "costUsd": 0.440296, "uncertainty": { "lower": 27.094, "upper": 33.482, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-6-luna" }, { "label": "Standard error", "value": "±3.19 points" }, { "label": "Mean latency", "value": "2,548 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "37.9%" }, { "label": "Business & Commercial", "value": "23.3%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "31.6%" }, { "label": "Criminal", "value": "26.3%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "53.3%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "79.3%" } ] }, { "id": "openai-gpt-6-sol", "label": "openai/gpt-6-sol", "model": "openai/gpt-6-sol", "provider": "OpenAI", "harness": null, "effort": "max", "score": 28.846, "costUsd": 4.897717, "uncertainty": { "lower": 25.697, "upper": 31.995, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "openai/gpt-6-sol" }, { "label": "Standard error", "value": "±3.15 points" }, { "label": "Mean latency", "value": "1,449 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "39.4%" }, { "label": "Business & Commercial", "value": "27.9%" }, { "label": "Civil Litigation", "value": "18.5%" }, { "label": "Constitutional / Civil Rights", "value": "18.4%" }, { "label": "Criminal", "value": "26.3%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "60.0%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "79.2%" } ] }, { "id": "poolside-laguna-m-1", "label": "poolside/laguna-m.1", "model": "poolside/laguna-m.1", "provider": "Poolside", "harness": null, "effort": null, "score": 2.404, "costUsd": 0.219839, "uncertainty": { "lower": 1.339, "upper": 3.469, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "poolside/laguna-m.1" }, { "label": "Standard error", "value": "±1.06 points" }, { "label": "Mean latency", "value": "466 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "3.0%" }, { "label": "Business & Commercial", "value": "4.7%" }, { "label": "Civil Litigation", "value": "0.0%" }, { "label": "Constitutional / Civil Rights", "value": "2.6%" }, { "label": "Criminal", "value": "0.0%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "3.3%" }, { "label": "Immigration", "value": "0.0%" }, { "label": "Weighted Pass", "value": "62.3%" } ] }, { "id": "poolside-laguna-xs-2", "label": "poolside/laguna-xs.2", "model": "poolside/laguna-xs.2", "provider": "Poolside", "harness": null, "effort": null, "score": 0.962, "costUsd": 0.107892, "uncertainty": { "lower": 0.2839999999999999, "upper": 1.6400000000000001, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "poolside/laguna-xs.2" }, { "label": "Standard error", "value": "±0.68 points" }, { "label": "Mean latency", "value": "352 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "1.5%" }, { "label": "Business & Commercial", "value": "0.0%" }, { "label": "Civil Litigation", "value": "3.7%" }, { "label": "Constitutional / Civil Rights", "value": "0.0%" }, { "label": "Criminal", "value": "0.0%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "0.0%" }, { "label": "Immigration", "value": "0.0%" }, { "label": "Weighted Pass", "value": "56.3%" } ] }, { "id": "tencent-hy4-preview", "label": "tencent/hy4-preview", "model": "tencent/hy4-preview", "provider": "Tencent", "harness": null, "effort": null, "score": 45.192, "costUsd": 0.846372, "uncertainty": { "lower": 41.733, "upper": 48.651, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "tencent/hy4-preview" }, { "label": "Standard error", "value": "±3.46 points" }, { "label": "Mean latency", "value": "3,840 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "54.5%" }, { "label": "Business & Commercial", "value": "44.2%" }, { "label": "Civil Litigation", "value": "59.3%" }, { "label": "Constitutional / Civil Rights", "value": "39.5%" }, { "label": "Criminal", "value": "36.8%" }, { "label": "Family", "value": "31.8%" }, { "label": "Health", "value": "66.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "85.2%" } ] }, { "id": "thinkingmachines-inkling", "label": "thinkingmachines/inkling", "model": "thinkingmachines/inkling", "provider": "Thinkingmachines", "harness": null, "effort": "0.99", "score": 28.365, "costUsd": 0.494044, "uncertainty": { "lower": 25.232, "upper": 31.497999999999998, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "thinkingmachines/inkling" }, { "label": "Standard error", "value": "±3.13 points" }, { "label": "Mean latency", "value": "869 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "31.8%" }, { "label": "Business & Commercial", "value": "25.6%" }, { "label": "Civil Litigation", "value": "33.3%" }, { "label": "Constitutional / Civil Rights", "value": "31.6%" }, { "label": "Criminal", "value": "21.1%" }, { "label": "Family", "value": "13.6%" }, { "label": "Health", "value": "43.3%" }, { "label": "Immigration", "value": "36.4%" }, { "label": "Weighted Pass", "value": "80.2%" } ] }, { "id": "thinkingmachines-inkling-small", "label": "thinkingmachines/inkling-small", "model": "thinkingmachines/inkling-small", "provider": "Thinkingmachines", "harness": null, "effort": "0.99", "score": 25.481, "costUsd": 0.141548, "uncertainty": { "lower": 22.452, "upper": 28.51, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "thinkingmachines/inkling-small" }, { "label": "Standard error", "value": "±3.03 points" }, { "label": "Mean latency", "value": "775 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "28.8%" }, { "label": "Business & Commercial", "value": "34.9%" }, { "label": "Civil Litigation", "value": "22.2%" }, { "label": "Constitutional / Civil Rights", "value": "10.5%" }, { "label": "Criminal", "value": "24.6%" }, { "label": "Family", "value": "4.5%" }, { "label": "Health", "value": "30.0%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "78.9%" } ] }, { "id": "xiaomi-mimo-v2-5", "label": "xiaomi/mimo-v2.5", "model": "xiaomi/mimo-v2.5", "provider": "Xiaomi", "harness": null, "effort": null, "score": 9.135, "costUsd": 0.033318, "uncertainty": { "lower": 7.133, "upper": 11.137, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.5" }, { "label": "Standard error", "value": "±2.00 points" }, { "label": "Mean latency", "value": "391 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "10.6%" }, { "label": "Business & Commercial", "value": "9.3%" }, { "label": "Civil Litigation", "value": "11.1%" }, { "label": "Constitutional / Civil Rights", "value": "7.9%" }, { "label": "Criminal", "value": "7.0%" }, { "label": "Family", "value": "0.0%" }, { "label": "Health", "value": "6.7%" }, { "label": "Immigration", "value": "9.1%" }, { "label": "Weighted Pass", "value": "68.2%" } ] }, { "id": "xiaomi-mimo-v2-5-pro", "label": "xiaomi/mimo-v2.5-pro", "model": "xiaomi/mimo-v2.5-pro", "provider": "Xiaomi", "harness": null, "effort": null, "score": 15.865, "costUsd": 0.082648, "uncertainty": { "lower": 13.326, "upper": 18.404, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.5-pro" }, { "label": "Standard error", "value": "±2.54 points" }, { "label": "Mean latency", "value": "416 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "18.2%" }, { "label": "Business & Commercial", "value": "11.6%" }, { "label": "Civil Litigation", "value": "11.1%" }, { "label": "Constitutional / Civil Rights", "value": "13.2%" }, { "label": "Criminal", "value": "14.0%" }, { "label": "Family", "value": "4.5%" }, { "label": "Health", "value": "23.3%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "76.0%" } ] }, { "id": "xiaomi-mimo-v2-6-flash", "label": "xiaomi/mimo-v2.6-flash", "model": "xiaomi/mimo-v2.6-flash", "provider": "Xiaomi", "harness": null, "effort": null, "score": 37.981, "costUsd": 0.070424, "uncertainty": { "lower": 34.608000000000004, "upper": 41.354, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.6-flash" }, { "label": "Standard error", "value": "±3.37 points" }, { "label": "Mean latency", "value": "920 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "47.0%" }, { "label": "Business & Commercial", "value": "37.2%" }, { "label": "Civil Litigation", "value": "29.6%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "29.8%" }, { "label": "Family", "value": "31.8%" }, { "label": "Health", "value": "53.3%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "79.6%" } ] }, { "id": "xiaomi-mimo-v2-6-pro", "label": "xiaomi/mimo-v2.6-pro", "model": "xiaomi/mimo-v2.6-pro", "provider": "Xiaomi", "harness": null, "effort": null, "score": 47.115, "costUsd": 0.181201, "uncertainty": { "lower": 43.646, "upper": 50.584, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.6-pro" }, { "label": "Standard error", "value": "±3.47 points" }, { "label": "Mean latency", "value": "1,816 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "59.1%" }, { "label": "Business & Commercial", "value": "44.2%" }, { "label": "Civil Litigation", "value": "40.7%" }, { "label": "Constitutional / Civil Rights", "value": "47.4%" }, { "label": "Criminal", "value": "43.9%" }, { "label": "Family", "value": "27.3%" }, { "label": "Health", "value": "70.0%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "85.5%" } ] }, { "id": "zai-glm-5-1", "label": "zai/glm-5.1", "model": "zai/glm-5.1", "provider": "Zhipu AI", "harness": null, "effort": null, "score": 27.885, "costUsd": 0.385737, "uncertainty": { "lower": 24.768, "upper": 31.002000000000002, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "zai/glm-5.1" }, { "label": "Standard error", "value": "±3.12 points" }, { "label": "Mean latency", "value": "326 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "31.8%" }, { "label": "Business & Commercial", "value": "23.3%" }, { "label": "Civil Litigation", "value": "22.2%" }, { "label": "Constitutional / Civil Rights", "value": "34.2%" }, { "label": "Criminal", "value": "29.8%" }, { "label": "Family", "value": "9.1%" }, { "label": "Health", "value": "36.7%" }, { "label": "Immigration", "value": "18.2%" }, { "label": "Weighted Pass", "value": "77.8%" } ] }, { "id": "zai-glm-5-2", "label": "zai/glm-5.2", "model": "zai/glm-5.2", "provider": "Zhipu AI", "harness": null, "effort": "max", "score": 31.25, "costUsd": 0.889209, "uncertainty": { "lower": 28.028, "upper": 34.472, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "zai/glm-5.2" }, { "label": "Standard error", "value": "±3.22 points" }, { "label": "Mean latency", "value": "1,023 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "39.4%" }, { "label": "Business & Commercial", "value": "27.9%" }, { "label": "Civil Litigation", "value": "25.9%" }, { "label": "Constitutional / Civil Rights", "value": "28.9%" }, { "label": "Criminal", "value": "22.8%" }, { "label": "Family", "value": "22.7%" }, { "label": "Health", "value": "50.0%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "82.9%" } ] }, { "id": "zai-glm-5-3", "label": "zai/glm-5.3", "model": "zai/glm-5.3", "provider": "Zhipu AI", "harness": null, "effort": "max", "score": 49.038, "costUsd": 2.241388, "uncertainty": { "lower": 45.562999999999995, "upper": 52.513, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "zai/glm-5.3" }, { "label": "Standard error", "value": "±3.48 points" }, { "label": "Mean latency", "value": "3,110 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "62.1%" }, { "label": "Business & Commercial", "value": "46.5%" }, { "label": "Civil Litigation", "value": "55.6%" }, { "label": "Constitutional / Civil Rights", "value": "39.5%" }, { "label": "Criminal", "value": "38.6%" }, { "label": "Family", "value": "31.8%" }, { "label": "Health", "value": "76.7%" }, { "label": "Immigration", "value": "27.3%" }, { "label": "Weighted Pass", "value": "87.7%" } ] }, { "id": "zai-glm-5-3-flash", "label": "zai/glm-5.3-flash", "model": "zai/glm-5.3-flash", "provider": "Zhipu AI", "harness": null, "effort": "max", "score": 45.192, "costUsd": 0.120196, "uncertainty": { "lower": 41.733, "upper": 48.651, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/legal_research", "details": [ { "label": "Publisher model id", "value": "zai/glm-5.3-flash" }, { "label": "Standard error", "value": "±3.46 points" }, { "label": "Mean latency", "value": "3,067 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Administrative / Regulatory", "value": "57.6%" }, { "label": "Business & Commercial", "value": "44.2%" }, { "label": "Civil Litigation", "value": "40.7%" }, { "label": "Constitutional / Civil Rights", "value": "44.7%" }, { "label": "Criminal", "value": "33.3%" }, { "label": "Family", "value": "27.3%" }, { "label": "Health", "value": "73.3%" }, { "label": "Immigration", "value": "45.5%" }, { "label": "Weighted Pass", "value": "87.5%" } ] } ] } }