{ "schemaVersion": 1, "benchmark": { "id": "vals-index", "name": "Vals Index · Vals AI", "version": "2 · private test set", "category": "general", "question": "Which model does the most economically valuable work?", "summary": "A GDP-weighted average of agentic performance across finance, coding, and legal tasks.", "source": { "name": "Vals AI", "url": "https://www.vals.ai/benchmarks/vals_index", "methodologyUrl": "https://www.vals.ai/about" }, "measure": "The publisher's composite of seven evaluations, weighted by each sector's share of U.S. GDP: finance 8.0, coding 5.6, and legal 1.2, over a denominator of 14.8.", "comparisonRule": "Compare within index version 2. Versions 1 and 2 use different component benchmarks, so their scores are not a series.", "limitations": [ "This is the publisher's composite, not an aicharts ranking. The sector weights are a deliberate simplification of how AI reaches the economy.", "The Code Migration component scores a fixed 60-task subset of the published 120-task run, not the full benchmark.", "Five of the seven components are private evaluations that no independent party can reproduce.", "Model names are the identifiers Vals publishes, not aicharts canonical names." ], "coverage": "charted", "tags": [ "composite", "economic impact", "agentic", "finance", "legal", "coding", "GDP", "Vals AI", "independent evaluator" ] }, "dataset": { "benchmarkId": "vals-index", "version": "2 · private test set", "score": { "label": "Accuracy", "unit": "%", "direction": "higher", "minimum": 0, "maximum": 100 }, "source": { "name": "Vals AI · Vals Index", "url": "https://www.vals.ai/benchmarks/vals_index", "retrievedAt": "2026-09-28T18:16:23.422Z", "revision": "vals_index v2" }, "observedAt": "2026-09-27T00:00:00.000Z", "evidenceLabel": "Independent evaluator · private test set", "configurationLabel": "Model at the effort Vals selected", "comparabilityNote": "One publisher-defined composite. Cost is the average dollar cost of one test across the weighted components, not the cost of a task you would run. Model names are the identifiers Vals publishes, not aicharts canonical names.", "costLabel": "Average cost per test (USD)", "points": [ { "id": "alibaba-qwen3-6-plus", "label": "alibaba/qwen3.6-plus", "model": "alibaba/qwen3.6-plus", "provider": "Alibaba", "harness": null, "effort": null, "score": 31.979, "costUsd": 2.618074, "uncertainty": { "lower": 31.003, "upper": 32.955, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.6-plus" }, { "label": "Standard error", "value": "±0.98 points" }, { "label": "Mean latency", "value": "1,260 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "40.8%" }, { "label": "EMB", "value": "32.9%" }, { "label": "Terminal-Bench 2.1", "value": "53.2%" }, { "label": "Vibe Code Bench", "value": "25.6%" }, { "label": "Code Migration", "value": "11.6%" }, { "label": "Legal Research", "value": "14.9%" }, { "label": "HLAB", "value": "1.3%" } ] }, { "id": "alibaba-qwen3-7-max", "label": "alibaba/qwen3.7-max", "model": "alibaba/qwen3.7-max", "provider": "Alibaba", "harness": null, "effort": null, "score": 44.769, "costUsd": 2.850897, "uncertainty": { "lower": 43.690999999999995, "upper": 45.847, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.7-max" }, { "label": "Standard error", "value": "±1.08 points" }, { "label": "Mean latency", "value": "1,782 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "47.8%" }, { "label": "EMB", "value": "57.0%" }, { "label": "Terminal-Bench 2.1", "value": "61.0%" }, { "label": "Vibe Code Bench", "value": "47.7%" }, { "label": "Code Migration", "value": "12.9%" }, { "label": "Legal Research", "value": "25.5%" }, { "label": "HLAB", "value": "1.7%" } ] }, { "id": "alibaba-qwen3-7-plus", "label": "alibaba/qwen3.7-plus", "model": "alibaba/qwen3.7-plus", "provider": "Alibaba", "harness": null, "effort": null, "score": 38.645, "costUsd": 0.439693, "uncertainty": { "lower": 37.474000000000004, "upper": 39.816, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.7-plus" }, { "label": "Standard error", "value": "±1.17 points" }, { "label": "Mean latency", "value": "1,308 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "38.2%" }, { "label": "EMB", "value": "49.3%" }, { "label": "Terminal-Bench 2.1", "value": "52.8%" }, { "label": "Vibe Code Bench", "value": "46.4%" }, { "label": "Code Migration", "value": "14.3%" }, { "label": "Legal Research", "value": "16.3%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "alibaba-qwen3-8-27b", "label": "alibaba/qwen3.8-27b", "model": "alibaba/qwen3.8-27b", "provider": "Alibaba", "harness": null, "effort": "xhigh", "score": 48.485, "costUsd": 8.479089, "uncertainty": { "lower": 47.031, "upper": 49.939, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.8-27b" }, { "label": "Standard error", "value": "±1.45 points" }, { "label": "Mean latency", "value": "3,635 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "48.6%" }, { "label": "EMB", "value": "59.7%" }, { "label": "Terminal-Bench 2.1", "value": "58.4%" }, { "label": "Vibe Code Bench", "value": "64.8%" }, { "label": "Code Migration", "value": "14.0%" }, { "label": "Legal Research", "value": "36.1%" }, { "label": "HLAB", "value": "11.3%" } ] }, { "id": "alibaba-qwen3-8-max", "label": "alibaba/qwen3.8-max", "model": "alibaba/qwen3.8-max", "provider": "Alibaba", "harness": null, "effort": null, "score": 51.844, "costUsd": 3.996716, "uncertainty": { "lower": 50.556000000000004, "upper": 53.132, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "alibaba/qwen3.8-max" }, { "label": "Standard error", "value": "±1.29 points" }, { "label": "Mean latency", "value": "5,446 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "50.6%" }, { "label": "EMB", "value": "60.1%" }, { "label": "Terminal-Bench 2.1", "value": "67.4%" }, { "label": "Vibe Code Bench", "value": "64.7%" }, { "label": "Code Migration", "value": "23.2%" }, { "label": "Legal Research", "value": "47.6%" }, { "label": "HLAB", "value": "10.4%" } ] }, { "id": "ant-ling-3-0-flash-2607", "label": "ant/ling-3.0-flash-2607", "model": "ant/ling-3.0-flash-2607", "provider": "Ant", "harness": null, "effort": null, "score": 21.699, "costUsd": 0.104406, "uncertainty": { "lower": 20.916, "upper": 22.482000000000003, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "ant/ling-3.0-flash-2607" }, { "label": "Standard error", "value": "±0.78 points" }, { "label": "Mean latency", "value": "2,983 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "30.3%" }, { "label": "EMB", "value": "25.0%" }, { "label": "Terminal-Bench 2.1", "value": "50.2%" }, { "label": "Vibe Code Bench", "value": "2.9%" }, { "label": "Code Migration", "value": "0.0%" }, { "label": "Legal Research", "value": "0.0%" }, { "label": "HLAB", "value": "1.3%" } ] }, { "id": "ant-ling-3-0-flash-af-rc3", "label": "ant/ling-3.0-flash-af-rc3", "model": "ant/ling-3.0-flash-af-rc3", "provider": "Ant", "harness": null, "effort": null, "score": 32.553, "costUsd": 0.109173, "uncertainty": { "lower": 31.586999999999996, "upper": 33.519, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "ant/ling-3.0-flash-af-rc3" }, { "label": "Standard error", "value": "±0.97 points" }, { "label": "Mean latency", "value": "1,073 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "54.9%" }, { "label": "EMB", "value": "36.8%" }, { "label": "Terminal-Bench 2.1", "value": "50.2%" }, { "label": "Vibe Code Bench", "value": "3.4%" }, { "label": "Code Migration", "value": "2.9%" }, { "label": "Legal Research", "value": "15.4%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "anthropic-claude-fable-5", "label": "anthropic/claude-fable-5", "model": "anthropic/claude-fable-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 66.036, "costUsd": 28.725071, "uncertainty": { "lower": 65.008, "upper": 67.06400000000001, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-fable-5" }, { "label": "Standard error", "value": "±1.03 points" }, { "label": "Mean latency", "value": "2,271 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "56.3%" }, { "label": "EMB", "value": "73.7%" }, { "label": "Terminal-Bench 2.1", "value": "80.5%" }, { "label": "Vibe Code Bench", "value": "90.4%" }, { "label": "Code Migration", "value": "54.6%" }, { "label": "Legal Research", "value": "49.5%" }, { "label": "HLAB", "value": "11.3%" } ] }, { "id": "anthropic-claude-fable-5-1", "label": "anthropic/claude-fable-5-1", "model": "anthropic/claude-fable-5-1", "provider": "Anthropic", "harness": null, "effort": "max", "score": 68.825, "costUsd": 28.916238, "uncertainty": { "lower": 67.748, "upper": 69.902, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-fable-5-1" }, { "label": "Standard error", "value": "±1.08 points" }, { "label": "Mean latency", "value": "4,576 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "58.9%" }, { "label": "EMB", "value": "76.7%" }, { "label": "Terminal-Bench 2.1", "value": "85.0%" }, { "label": "Vibe Code Bench", "value": "90.3%" }, { "label": "Code Migration", "value": "60.0%" }, { "label": "Legal Research", "value": "55.3%" }, { "label": "HLAB", "value": "6.7%" } ] }, { "id": "anthropic-claude-haiku-4-5-20251001-thinking", "label": "anthropic/claude-haiku-4-5-20251001-thinking", "model": "anthropic/claude-haiku-4-5-20251001-thinking", "provider": "Anthropic", "harness": null, "effort": null, "score": 22.898, "costUsd": 0.697729, "uncertainty": { "lower": 22.059, "upper": 23.737, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-haiku-4-5-20251001-thinking" }, { "label": "Standard error", "value": "±0.84 points" }, { "label": "Mean latency", "value": "500 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "31.0%" }, { "label": "EMB", "value": "22.7%" }, { "label": "Terminal-Bench 2.1", "value": "43.8%" }, { "label": "Vibe Code Bench", "value": "11.4%" }, { "label": "Code Migration", "value": "7.6%" }, { "label": "Legal Research", "value": "10.6%" }, { "label": "HLAB", "value": "0.8%" } ] }, { "id": "anthropic-claude-opus-4-7", "label": "anthropic/claude-opus-4-7", "model": "anthropic/claude-opus-4-7", "provider": "Anthropic", "harness": null, "effort": "high", "score": 56.114, "costUsd": 12.760253, "uncertainty": { "lower": 54.9, "upper": 57.327999999999996, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-4-7" }, { "label": "Standard error", "value": "±1.21 points" }, { "label": "Mean latency", "value": "1,706 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "51.5%" }, { "label": "EMB", "value": "63.7%" }, { "label": "Terminal-Bench 2.1", "value": "68.5%" }, { "label": "Vibe Code Bench", "value": "71.0%" }, { "label": "Code Migration", "value": "43.9%" }, { "label": "Legal Research", "value": "38.5%" }, { "label": "HLAB", "value": "6.7%" } ] }, { "id": "anthropic-claude-opus-4-8", "label": "anthropic/claude-opus-4-8", "model": "anthropic/claude-opus-4-8", "provider": "Anthropic", "harness": null, "effort": "max", "score": 60.908, "costUsd": 12.66685, "uncertainty": { "lower": 59.881, "upper": 61.935, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-4-8" }, { "label": "Standard error", "value": "±1.03 points" }, { "label": "Mean latency", "value": "2,182 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "53.9%" }, { "label": "EMB", "value": "69.4%" }, { "label": "Terminal-Bench 2.1", "value": "71.9%" }, { "label": "Vibe Code Bench", "value": "82.7%" }, { "label": "Code Migration", "value": "46.9%" }, { "label": "Legal Research", "value": "43.8%" }, { "label": "HLAB", "value": "9.6%" } ] }, { "id": "anthropic-claude-opus-5", "label": "anthropic/claude-opus-5", "model": "anthropic/claude-opus-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 67.213, "costUsd": 18.810212, "uncertainty": { "lower": 66.235, "upper": 68.19099999999999, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-5" }, { "label": "Standard error", "value": "±0.98 points" }, { "label": "Mean latency", "value": "3,349 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "58.6%" }, { "label": "EMB", "value": "73.6%" }, { "label": "Terminal-Bench 2.1", "value": "84.6%" }, { "label": "Vibe Code Bench", "value": "88.4%" }, { "label": "Code Migration", "value": "56.7%" }, { "label": "Legal Research", "value": "55.3%" }, { "label": "HLAB", "value": "6.7%" } ] }, { "id": "anthropic-claude-opus-5-5", "label": "anthropic/claude-opus-5-5", "model": "anthropic/claude-opus-5-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 69.689, "costUsd": 32.766011, "uncertainty": { "lower": 68.752, "upper": 70.62599999999999, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-opus-5-5" }, { "label": "Standard error", "value": "±0.94 points" }, { "label": "Mean latency", "value": "4,321 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "58.6%" }, { "label": "EMB", "value": "75.9%" }, { "label": "Terminal-Bench 2.1", "value": "87.6%" }, { "label": "Vibe Code Bench", "value": "90.3%" }, { "label": "Code Migration", "value": "68.9%" }, { "label": "Legal Research", "value": "50.5%" }, { "label": "HLAB", "value": "3.8%" } ] }, { "id": "anthropic-claude-sonnet-4-6", "label": "anthropic/claude-sonnet-4-6", "model": "anthropic/claude-sonnet-4-6", "provider": "Anthropic", "harness": null, "effort": "max", "score": 50.593, "costUsd": 8.404689, "uncertainty": { "lower": 49.358000000000004, "upper": 51.828, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-sonnet-4-6" }, { "label": "Standard error", "value": "±1.24 points" }, { "label": "Mean latency", "value": "2,086 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "51.0%" }, { "label": "EMB", "value": "60.2%" }, { "label": "Terminal-Bench 2.1", "value": "57.3%" }, { "label": "Vibe Code Bench", "value": "51.5%" }, { "label": "Code Migration", "value": "40.1%" }, { "label": "Legal Research", "value": "38.5%" }, { "label": "HLAB", "value": "5.0%" } ] }, { "id": "anthropic-claude-sonnet-5", "label": "anthropic/claude-sonnet-5", "model": "anthropic/claude-sonnet-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 59.61, "costUsd": 11.740516, "uncertainty": { "lower": 58.463, "upper": 60.757, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-sonnet-5" }, { "label": "Standard error", "value": "±1.15 points" }, { "label": "Mean latency", "value": "2,746 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "53.9%" }, { "label": "EMB", "value": "66.3%" }, { "label": "Terminal-Bench 2.1", "value": "74.5%" }, { "label": "Vibe Code Bench", "value": "81.3%" }, { "label": "Code Migration", "value": "44.1%" }, { "label": "Legal Research", "value": "41.8%" }, { "label": "HLAB", "value": "5.0%" } ] }, { "id": "anthropic-claude-sonnet-5-5", "label": "anthropic/claude-sonnet-5-5", "model": "anthropic/claude-sonnet-5-5", "provider": "Anthropic", "harness": null, "effort": "max", "score": 69.219, "costUsd": 20.803442, "uncertainty": { "lower": 68.25699999999999, "upper": 70.181, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "anthropic/claude-sonnet-5-5" }, { "label": "Standard error", "value": "±0.96 points" }, { "label": "Mean latency", "value": "4,207 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "58.1%" }, { "label": "EMB", "value": "75.7%" }, { "label": "Terminal-Bench 2.1", "value": "83.1%" }, { "label": "Vibe Code Bench", "value": "92.4%" }, { "label": "Code Migration", "value": "70.1%" }, { "label": "Legal Research", "value": "48.1%" }, { "label": "HLAB", "value": "2.9%" } ] }, { "id": "deepseek-deepseek-v4-1-flash", "label": "deepseek/deepseek-v4.1-flash", "model": "deepseek/deepseek-v4.1-flash", "provider": "DeepSeek", "harness": null, "effort": "high", "score": 57.86, "costUsd": 0.303291, "uncertainty": { "lower": 56.7, "upper": 59.019999999999996, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4.1-flash" }, { "label": "Standard error", "value": "±1.16 points" }, { "label": "Mean latency", "value": "1,534 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "53.5%" }, { "label": "EMB", "value": "57.2%" }, { "label": "Terminal-Bench 2.1", "value": "74.5%" }, { "label": "Vibe Code Bench", "value": "84.7%" }, { "label": "Code Migration", "value": "46.9%" }, { "label": "Legal Research", "value": "41.3%" }, { "label": "HLAB", "value": "6.7%" } ] }, { "id": "deepseek-deepseek-v4-flash-0731", "label": "deepseek/deepseek-v4-flash-0731", "model": "deepseek/deepseek-v4-flash-0731", "provider": "DeepSeek", "harness": null, "effort": "high", "score": 53.568, "costUsd": 0.839765, "uncertainty": { "lower": 52.358, "upper": 54.778, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4-flash-0731" }, { "label": "Standard error", "value": "±1.21 points" }, { "label": "Mean latency", "value": "2,095 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "49.5%" }, { "label": "EMB", "value": "57.0%" }, { "label": "Terminal-Bench 2.1", "value": "67.0%" }, { "label": "Vibe Code Bench", "value": "74.7%" }, { "label": "Code Migration", "value": "42.3%" }, { "label": "Legal Research", "value": "30.3%" }, { "label": "HLAB", "value": "8.3%" } ] }, { "id": "deepseek-deepseek-v4-pro", "label": "deepseek/deepseek-v4-pro", "model": "deepseek/deepseek-v4-pro", "provider": "DeepSeek", "harness": null, "effort": "max", "score": 42.888, "costUsd": 0.940927, "uncertainty": { "lower": 41.693999999999996, "upper": 44.082, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4-pro" }, { "label": "Standard error", "value": "±1.19 points" }, { "label": "Mean latency", "value": "1,427 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "44.1%" }, { "label": "EMB", "value": "51.6%" }, { "label": "Terminal-Bench 2.1", "value": "50.2%" }, { "label": "Vibe Code Bench", "value": "49.9%" }, { "label": "Code Migration", "value": "26.2%" }, { "label": "Legal Research", "value": "23.1%" }, { "label": "HLAB", "value": "3.8%" } ] }, { "id": "deepseek-deepseek-v4-pro-0813", "label": "deepseek/deepseek-v4-pro-0813", "model": "deepseek/deepseek-v4-pro-0813", "provider": "DeepSeek", "harness": null, "effort": "max", "score": 52.368, "costUsd": 3.282301, "uncertainty": { "lower": 51.232, "upper": 53.504000000000005, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "deepseek/deepseek-v4-pro-0813" }, { "label": "Standard error", "value": "±1.14 points" }, { "label": "Mean latency", "value": "3,498 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "50.4%" }, { "label": "EMB", "value": "52.8%" }, { "label": "Terminal-Bench 2.1", "value": "54.7%" }, { "label": "Vibe Code Bench", "value": "82.3%" }, { "label": "Code Migration", "value": "41.5%" }, { "label": "Legal Research", "value": "40.9%" }, { "label": "HLAB", "value": "7.5%" } ] }, { "id": "fireworks-nemotron-lightning-3p5-30b-a3b", "label": "fireworks/nemotron-lightning-3p5-30b-a3b", "model": "fireworks/nemotron-lightning-3p5-30b-a3b", "provider": "Fireworks AI", "harness": null, "effort": null, "score": 11.494, "costUsd": 0.09345, "uncertainty": { "lower": 10.884, "upper": 12.104, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "fireworks/nemotron-lightning-3p5-30b-a3b" }, { "label": "Standard error", "value": "±0.61 points" }, { "label": "Mean latency", "value": "1,345 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "18.5%" }, { "label": "EMB", "value": "17.0%" }, { "label": "Terminal-Bench 2.1", "value": "10.9%" }, { "label": "Vibe Code Bench", "value": "0.0%" }, { "label": "Code Migration", "value": "3.3%" }, { "label": "Legal Research", "value": "2.9%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "google-gemini-3-1-flash-lite-preview", "label": "google/gemini-3.1-flash-lite-preview", "model": "google/gemini-3.1-flash-lite-preview", "provider": "Google", "harness": null, "effort": "high", "score": 15.457, "costUsd": 0.136314, "uncertainty": { "lower": 15.040000000000001, "upper": 15.874, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.1-flash-lite-preview" }, { "label": "Standard error", "value": "±0.42 points" }, { "label": "Mean latency", "value": "163 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "30.0%" }, { "label": "EMB", "value": "8.6%" }, { "label": "Terminal-Bench 2.1", "value": "34.1%" }, { "label": "Vibe Code Bench", "value": "0.0%" }, { "label": "Code Migration", "value": "4.6%" }, { "label": "Legal Research", "value": "3.4%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "google-gemini-3-1-pro-preview", "label": "google/gemini-3.1-pro-preview", "model": "google/gemini-3.1-pro-preview", "provider": "Google", "harness": null, "effort": "high", "score": 41.903, "costUsd": 1.944534, "uncertainty": { "lower": 40.73, "upper": 43.076, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.1-pro-preview" }, { "label": "Standard error", "value": "±1.17 points" }, { "label": "Mean latency", "value": "560 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "43.0%" }, { "label": "EMB", "value": "52.6%" }, { "label": "Terminal-Bench 2.1", "value": "70.8%" }, { "label": "Vibe Code Bench", "value": "32.0%" }, { "label": "Code Migration", "value": "17.9%" }, { "label": "Legal Research", "value": "20.7%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "google-gemini-3-5-flash", "label": "google/gemini-3.5-flash", "model": "google/gemini-3.5-flash", "provider": "Google", "harness": null, "effort": "high", "score": 53.079, "costUsd": 2.914646, "uncertainty": { "lower": 51.957, "upper": 54.201, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.5-flash" }, { "label": "Standard error", "value": "±1.12 points" }, { "label": "Mean latency", "value": "661 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "57.9%" }, { "label": "EMB", "value": "63.5%" }, { "label": "Terminal-Bench 2.1", "value": "74.2%" }, { "label": "Vibe Code Bench", "value": "48.7%" }, { "label": "Code Migration", "value": "27.1%" }, { "label": "Legal Research", "value": "30.8%" }, { "label": "HLAB", "value": "2.5%" } ] }, { "id": "google-gemini-3-5-flash-lite", "label": "google/gemini-3.5-flash-lite", "model": "google/gemini-3.5-flash-lite", "provider": "Google", "harness": null, "effort": "high", "score": 36.711, "costUsd": 0.543466, "uncertainty": { "lower": 35.646, "upper": 37.775999999999996, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.5-flash-lite" }, { "label": "Standard error", "value": "±1.06 points" }, { "label": "Mean latency", "value": "461 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "47.4%" }, { "label": "EMB", "value": "43.2%" }, { "label": "Terminal-Bench 2.1", "value": "50.2%" }, { "label": "Vibe Code Bench", "value": "37.2%" }, { "label": "Code Migration", "value": "4.9%" }, { "label": "Legal Research", "value": "13.9%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "google-gemini-3-6-flash", "label": "google/gemini-3.6-flash", "model": "google/gemini-3.6-flash", "provider": "Google", "harness": null, "effort": "high", "score": 55.354, "costUsd": 3.02952, "uncertainty": { "lower": 54.262, "upper": 56.446, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.6-flash" }, { "label": "Standard error", "value": "±1.09 points" }, { "label": "Mean latency", "value": "1,328 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "56.3%" }, { "label": "EMB", "value": "65.4%" }, { "label": "Terminal-Bench 2.1", "value": "73.8%" }, { "label": "Vibe Code Bench", "value": "64.0%" }, { "label": "Code Migration", "value": "31.2%" }, { "label": "Legal Research", "value": "25.0%" }, { "label": "HLAB", "value": "3.3%" } ] }, { "id": "google-gemini-3-7-flash", "label": "google/gemini-3.7-flash", "model": "google/gemini-3.7-flash", "provider": "Google", "harness": null, "effort": "high", "score": 59.307, "costUsd": 4.169542, "uncertainty": { "lower": 58.242000000000004, "upper": 60.372, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.7-flash" }, { "label": "Standard error", "value": "±1.06 points" }, { "label": "Mean latency", "value": "2,593 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "59.0%" }, { "label": "EMB", "value": "71.3%" }, { "label": "Terminal-Bench 2.1", "value": "77.5%" }, { "label": "Vibe Code Bench", "value": "70.4%" }, { "label": "Code Migration", "value": "29.0%" }, { "label": "Legal Research", "value": "34.6%" }, { "label": "HLAB", "value": "8.8%" } ] }, { "id": "google-gemini-3-8-flash", "label": "google/gemini-3.8-flash", "model": "google/gemini-3.8-flash", "provider": "Google", "harness": null, "effort": "high", "score": 62.252, "costUsd": 5.391952, "uncertainty": { "lower": 61.243, "upper": 63.261, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3.8-flash" }, { "label": "Standard error", "value": "±1.01 points" }, { "label": "Mean latency", "value": "2,693 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "61.4%" }, { "label": "EMB", "value": "72.2%" }, { "label": "Terminal-Bench 2.1", "value": "81.3%" }, { "label": "Vibe Code Bench", "value": "78.7%" }, { "label": "Code Migration", "value": "31.6%" }, { "label": "Legal Research", "value": "38.9%" }, { "label": "HLAB", "value": "10.0%" } ] }, { "id": "google-gemini-3-flash-preview", "label": "google/gemini-3-flash-preview", "model": "google/gemini-3-flash-preview", "provider": "Google", "harness": null, "effort": "high", "score": 29.444, "costUsd": 0.409423, "uncertainty": { "lower": 28.461, "upper": 30.427, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "google/gemini-3-flash-preview" }, { "label": "Standard error", "value": "±0.98 points" }, { "label": "Mean latency", "value": "348 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "42.6%" }, { "label": "EMB", "value": "26.0%" }, { "label": "Terminal-Bench 2.1", "value": "53.9%" }, { "label": "Vibe Code Bench", "value": "20.2%" }, { "label": "Code Migration", "value": "6.5%" }, { "label": "Legal Research", "value": "18.3%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "grok-grok-4-20-0309-reasoning", "label": "grok/grok-4.20-0309-reasoning", "model": "grok/grok-4.20-0309-reasoning", "provider": "SpaceXAI", "harness": null, "effort": null, "score": 17.552, "costUsd": 0.569009, "uncertainty": { "lower": 17.026, "upper": 18.078, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.20-0309-reasoning" }, { "label": "Standard error", "value": "±0.53 points" }, { "label": "Mean latency", "value": "350 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "28.5%" }, { "label": "EMB", "value": "11.7%" }, { "label": "Terminal-Bench 2.1", "value": "44.2%" }, { "label": "Vibe Code Bench", "value": "4.1%" }, { "label": "Code Migration", "value": "0.2%" }, { "label": "Legal Research", "value": "13.9%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "grok-grok-4-3", "label": "grok/grok-4.3", "model": "grok/grok-4.3", "provider": "SpaceXAI", "harness": null, "effort": "high", "score": 24.294, "costUsd": 0.676033, "uncertainty": { "lower": 23.481, "upper": 25.107, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.3" }, { "label": "Standard error", "value": "±0.81 points" }, { "label": "Mean latency", "value": "338 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "37.7%" }, { "label": "EMB", "value": "18.4%" }, { "label": "Terminal-Bench 2.1", "value": "41.9%" }, { "label": "Vibe Code Bench", "value": "19.4%" }, { "label": "Code Migration", "value": "5.9%" }, { "label": "Legal Research", "value": "15.4%" }, { "label": "HLAB", "value": "0.4%" } ] }, { "id": "grok-grok-4-5", "label": "grok/grok-4.5", "model": "grok/grok-4.5", "provider": "SpaceXAI", "harness": null, "effort": "high", "score": 51.528, "costUsd": 1.955641, "uncertainty": { "lower": 50.211, "upper": 52.845, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.5" }, { "label": "Standard error", "value": "±1.32 points" }, { "label": "Mean latency", "value": "838 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "48.3%" }, { "label": "EMB", "value": "52.9%" }, { "label": "Terminal-Bench 2.1", "value": "67.8%" }, { "label": "Vibe Code Bench", "value": "69.0%" }, { "label": "Code Migration", "value": "38.4%" }, { "label": "Legal Research", "value": "38.0%" }, { "label": "HLAB", "value": "12.9%" } ] }, { "id": "grok-grok-4-6", "label": "grok/grok-4.6", "model": "grok/grok-4.6", "provider": "SpaceXAI", "harness": null, "effort": "high", "score": 59.167, "costUsd": 4.335319, "uncertainty": { "lower": 57.962, "upper": 60.372, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.6" }, { "label": "Standard error", "value": "±1.21 points" }, { "label": "Mean latency", "value": "2,284 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "53.7%" }, { "label": "EMB", "value": "62.7%" }, { "label": "Terminal-Bench 2.1", "value": "78.3%" }, { "label": "Vibe Code Bench", "value": "76.2%" }, { "label": "Code Migration", "value": "44.6%" }, { "label": "Legal Research", "value": "48.1%" }, { "label": "HLAB", "value": "15.8%" } ] }, { "id": "grok-grok-4-7", "label": "grok/grok-4.7", "model": "grok/grok-4.7", "provider": "SpaceXAI", "harness": null, "effort": "xhigh", "score": 60.215, "costUsd": 11.161883, "uncertainty": { "lower": 59.136, "upper": 61.294000000000004, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "grok/grok-4.7" }, { "label": "Standard error", "value": "±1.08 points" }, { "label": "Mean latency", "value": "1,976 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "52.3%" }, { "label": "EMB", "value": "67.0%" }, { "label": "Terminal-Bench 2.1", "value": "73.4%" }, { "label": "Vibe Code Bench", "value": "86.2%" }, { "label": "Code Migration", "value": "43.2%" }, { "label": "Legal Research", "value": "47.1%" }, { "label": "HLAB", "value": "12.5%" } ] }, { "id": "inception-mercury-2-5", "label": "inception/mercury-2.5", "model": "inception/mercury-2.5", "provider": "Inception", "harness": null, "effort": "high", "score": 12.952, "costUsd": 0.300647, "uncertainty": { "lower": 12.478, "upper": 13.426, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "inception/mercury-2.5" }, { "label": "Standard error", "value": "±0.47 points" }, { "label": "Mean latency", "value": "355 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "18.6%" }, { "label": "EMB", "value": "8.8%" }, { "label": "Terminal-Bench 2.1", "value": "34.1%" }, { "label": "Vibe Code Bench", "value": "3.7%" }, { "label": "Code Migration", "value": "4.9%" }, { "label": "Legal Research", "value": "4.3%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "kimi-kimi-k2-5-thinking", "label": "kimi/kimi-k2.5-thinking", "model": "kimi/kimi-k2.5-thinking", "provider": "Moonshot AI", "harness": null, "effort": null, "score": 26.301, "costUsd": 0.511754, "uncertainty": { "lower": 25.352999999999998, "upper": 27.249, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "kimi/kimi-k2.5-thinking" }, { "label": "Standard error", "value": "±0.95 points" }, { "label": "Mean latency", "value": "2,131 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "35.8%" }, { "label": "EMB", "value": "28.5%" }, { "label": "Terminal-Bench 2.1", "value": "41.9%" }, { "label": "Vibe Code Bench", "value": "17.5%" }, { "label": "Code Migration", "value": "6.2%" }, { "label": "Legal Research", "value": "15.9%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "kimi-kimi-k2-6", "label": "kimi/kimi-k2.6", "model": "kimi/kimi-k2.6", "provider": "Moonshot AI", "harness": null, "effort": null, "score": 43.465, "costUsd": 1.849897, "uncertainty": { "lower": 42.291000000000004, "upper": 44.639, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "kimi/kimi-k2.6" }, { "label": "Standard error", "value": "±1.17 points" }, { "label": "Mean latency", "value": "2,392 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "44.9%" }, { "label": "EMB", "value": "57.9%" }, { "label": "Terminal-Bench 2.1", "value": "53.6%" }, { "label": "Vibe Code Bench", "value": "37.9%" }, { "label": "Code Migration", "value": "27.3%" }, { "label": "Legal Research", "value": "15.9%" }, { "label": "HLAB", "value": "1.7%" } ] }, { "id": "kimi-kimi-k3", "label": "kimi/kimi-k3", "model": "kimi/kimi-k3", "provider": "Moonshot AI", "harness": null, "effort": null, "score": 57.813, "costUsd": 6.467826, "uncertainty": { "lower": 56.755, "upper": 58.871, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "kimi/kimi-k3" }, { "label": "Standard error", "value": "±1.06 points" }, { "label": "Mean latency", "value": "4,174 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "54.4%" }, { "label": "EMB", "value": "66.4%" }, { "label": "Terminal-Bench 2.1", "value": "80.9%" }, { "label": "Vibe Code Bench", "value": "85.0%" }, { "label": "Code Migration", "value": "16.0%" }, { "label": "Legal Research", "value": "44.2%" }, { "label": "HLAB", "value": "10.8%" } ] }, { "id": "meta-muse-spark-1-1", "label": "meta/muse_spark_1_1", "model": "meta/muse_spark_1_1", "provider": "Meta", "harness": null, "effort": "xhigh", "score": 54.754, "costUsd": 1.250502, "uncertainty": { "lower": 53.605999999999995, "upper": 55.902, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_1" }, { "label": "Standard error", "value": "±1.15 points" }, { "label": "Mean latency", "value": "828 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "57.2%" }, { "label": "EMB", "value": "56.4%" }, { "label": "Terminal-Bench 2.1", "value": "69.3%" }, { "label": "Vibe Code Bench", "value": "72.2%" }, { "label": "Code Migration", "value": "30.7%" }, { "label": "Legal Research", "value": "38.0%" }, { "label": "HLAB", "value": "20.0%" } ] }, { "id": "meta-muse-spark-1-2", "label": "meta/muse_spark_1_2", "model": "meta/muse_spark_1_2", "provider": "Meta", "harness": null, "effort": "xhigh", "score": 57.053, "costUsd": 1.661592, "uncertainty": { "lower": 55.936, "upper": 58.169999999999995, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_2" }, { "label": "Standard error", "value": "±1.12 points" }, { "label": "Mean latency", "value": "1,009 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "60.6%" }, { "label": "EMB", "value": "57.0%" }, { "label": "Terminal-Bench 2.1", "value": "69.7%" }, { "label": "Vibe Code Bench", "value": "79.1%" }, { "label": "Code Migration", "value": "29.4%" }, { "label": "Legal Research", "value": "43.8%" }, { "label": "HLAB", "value": "25.4%" } ] }, { "id": "meta-muse-spark-1-3", "label": "meta/muse_spark_1_3", "model": "meta/muse_spark_1_3", "provider": "Meta", "harness": null, "effort": "xhigh", "score": 60.31, "costUsd": 2.899528, "uncertainty": { "lower": 59.234, "upper": 61.386, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_3" }, { "label": "Standard error", "value": "±1.08 points" }, { "label": "Mean latency", "value": "1,277 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "58.9%" }, { "label": "EMB", "value": "62.7%" }, { "label": "Terminal-Bench 2.1", "value": "72.3%" }, { "label": "Vibe Code Bench", "value": "82.9%" }, { "label": "Code Migration", "value": "42.2%" }, { "label": "Legal Research", "value": "40.9%" }, { "label": "HLAB", "value": "22.1%" } ] }, { "id": "meta-muse-spark-1-3-max", "label": "meta/muse_spark_1_3_max", "model": "meta/muse_spark_1_3_max", "provider": "Meta", "harness": null, "effort": "max", "score": 64.526, "costUsd": 3.404111, "uncertainty": { "lower": 63.296, "upper": 65.756, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "meta/muse_spark_1_3_max" }, { "label": "Standard error", "value": "±1.23 points" }, { "label": "Mean latency", "value": "1,230 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "60.0%" }, { "label": "EMB", "value": "67.4%" }, { "label": "Terminal-Bench 2.1", "value": "79.0%" }, { "label": "Vibe Code Bench", "value": "85.9%" }, { "label": "Code Migration", "value": "48.3%" }, { "label": "Legal Research", "value": "55.3%" }, { "label": "HLAB", "value": "23.8%" } ] }, { "id": "minimax-minimax-m2-7", "label": "minimax/MiniMax-M2.7", "model": "minimax/MiniMax-M2.7", "provider": "MiniMax", "harness": null, "effort": null, "score": 24.556, "costUsd": 0.87768, "uncertainty": { "lower": 23.624000000000002, "upper": 25.488, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "minimax/MiniMax-M2.7" }, { "label": "Standard error", "value": "±0.93 points" }, { "label": "Mean latency", "value": "904 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "27.9%" }, { "label": "EMB", "value": "28.5%" }, { "label": "Terminal-Bench 2.1", "value": "48.7%" }, { "label": "Vibe Code Bench", "value": "11.9%" }, { "label": "Code Migration", "value": "10.0%" }, { "label": "Legal Research", "value": "10.6%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "minimax-minimax-m3", "label": "minimax/MiniMax-M3", "model": "minimax/MiniMax-M3", "provider": "MiniMax", "harness": null, "effort": null, "score": 42.719, "costUsd": 2.495943, "uncertainty": { "lower": 41.521, "upper": 43.917, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "minimax/MiniMax-M3" }, { "label": "Standard error", "value": "±1.20 points" }, { "label": "Mean latency", "value": "2,115 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "48.3%" }, { "label": "EMB", "value": "47.8%" }, { "label": "Terminal-Bench 2.1", "value": "53.6%" }, { "label": "Vibe Code Bench", "value": "47.6%" }, { "label": "Code Migration", "value": "20.9%" }, { "label": "Legal Research", "value": "29.8%" }, { "label": "HLAB", "value": "4.2%" } ] }, { "id": "mistralai-mistral-medium-3-5", "label": "mistralai/mistral-medium-3.5", "model": "mistralai/mistral-medium-3.5", "provider": "Mistral AI", "harness": null, "effort": "high", "score": 17.947, "costUsd": 8.010266, "uncertainty": { "lower": 17.259999999999998, "upper": 18.634, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "mistralai/mistral-medium-3.5" }, { "label": "Standard error", "value": "±0.69 points" }, { "label": "Mean latency", "value": "3,446 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "32.1%" }, { "label": "EMB", "value": "11.2%" }, { "label": "Terminal-Bench 2.1", "value": "39.0%" }, { "label": "Vibe Code Bench", "value": "2.9%" }, { "label": "Code Migration", "value": "4.6%" }, { "label": "Legal Research", "value": "9.1%" }, { "label": "HLAB", "value": "0.4%" } ] }, { "id": "nvidia-nemotron-3-ultra-550b-a55b", "label": "nvidia/nemotron-3-ultra-550b-a55b", "model": "nvidia/nemotron-3-ultra-550b-a55b", "provider": "Nvidia", "harness": null, "effort": null, "score": 27.395, "costUsd": null, "uncertainty": { "lower": 26.445999999999998, "upper": 28.344, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "nvidia/nemotron-3-ultra-550b-a55b" }, { "label": "Standard error", "value": "±0.95 points" }, { "label": "Mean latency", "value": "1,948 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "37.7%" }, { "label": "EMB", "value": "31.7%" }, { "label": "Terminal-Bench 2.1", "value": "50.9%" }, { "label": "Vibe Code Bench", "value": "7.6%" }, { "label": "Code Migration", "value": "4.8%" }, { "label": "Legal Research", "value": "15.4%" }, { "label": "HLAB", "value": "0.4%" } ] }, { "id": "openai-gpt-5-4-mini-2026-03-17", "label": "openai/gpt-5.4-mini-2026-03-17", "model": "openai/gpt-5.4-mini-2026-03-17", "provider": "OpenAI", "harness": null, "effort": "xhigh", "score": 39.632, "costUsd": 1.380101, "uncertainty": { "lower": 38.196999999999996, "upper": 41.067, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.4-mini-2026-03-17" }, { "label": "Standard error", "value": "±1.44 points" }, { "label": "Mean latency", "value": "1,742 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "45.4%" }, { "label": "EMB", "value": "45.4%" }, { "label": "Terminal-Bench 2.1", "value": "54.7%" }, { "label": "Vibe Code Bench", "value": "48.0%" }, { "label": "Code Migration", "value": "13.0%" }, { "label": "Legal Research", "value": "12.5%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "openai-gpt-5-4-nano-2026-03-17", "label": "openai/gpt-5.4-nano-2026-03-17", "model": "openai/gpt-5.4-nano-2026-03-17", "provider": "OpenAI", "harness": null, "effort": "high", "score": 32.997, "costUsd": 0.398254, "uncertainty": { "lower": 31.721, "upper": 34.273, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.4-nano-2026-03-17" }, { "label": "Standard error", "value": "±1.28 points" }, { "label": "Mean latency", "value": "1,230 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "38.2%" }, { "label": "EMB", "value": "44.8%" }, { "label": "Terminal-Bench 2.1", "value": "41.6%" }, { "label": "Vibe Code Bench", "value": "26.1%" }, { "label": "Code Migration", "value": "14.2%" }, { "label": "Legal Research", "value": "6.3%" }, { "label": "HLAB", "value": "0.0%" } ] }, { "id": "openai-gpt-5-5", "label": "openai/gpt-5.5", "model": "openai/gpt-5.5", "provider": "OpenAI", "harness": null, "effort": "xhigh", "score": 57.411, "costUsd": 6.142389, "uncertainty": { "lower": 56.26, "upper": 58.562000000000005, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.5" }, { "label": "Standard error", "value": "±1.15 points" }, { "label": "Mean latency", "value": "1,251 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "51.8%" }, { "label": "EMB", "value": "64.5%" }, { "label": "Terminal-Bench 2.1", "value": "76.4%" }, { "label": "Vibe Code Bench", "value": "69.8%" }, { "label": "Code Migration", "value": "45.5%" }, { "label": "Legal Research", "value": "40.4%" }, { "label": "HLAB", "value": "3.8%" } ] }, { "id": "openai-gpt-5-6-luna", "label": "openai/gpt-5.6-luna", "model": "openai/gpt-5.6-luna", "provider": "OpenAI", "harness": null, "effort": "max", "score": 59.881, "costUsd": 0.778845, "uncertainty": { "lower": 58.794, "upper": 60.968, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.6-luna" }, { "label": "Standard error", "value": "±1.09 points" }, { "label": "Mean latency", "value": "1,396 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "55.0%" }, { "label": "EMB", "value": "67.1%" }, { "label": "Terminal-Bench 2.1", "value": "79.0%" }, { "label": "Vibe Code Bench", "value": "77.1%" }, { "label": "Code Migration", "value": "44.8%" }, { "label": "Legal Research", "value": "36.5%" }, { "label": "HLAB", "value": "1.3%" } ] }, { "id": "openai-gpt-5-6-sol", "label": "openai/gpt-5.6-sol", "model": "openai/gpt-5.6-sol", "provider": "OpenAI", "harness": null, "effort": "max", "score": 63.709, "costUsd": 14.205832, "uncertainty": { "lower": 62.649, "upper": 64.769, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.6-sol" }, { "label": "Standard error", "value": "±1.06 points" }, { "label": "Mean latency", "value": "1,944 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "53.8%" }, { "label": "EMB", "value": "72.3%" }, { "label": "Terminal-Bench 2.1", "value": "85.8%" }, { "label": "Vibe Code Bench", "value": "80.5%" }, { "label": "Code Migration", "value": "52.4%" }, { "label": "Legal Research", "value": "48.1%" }, { "label": "HLAB", "value": "2.5%" } ] }, { "id": "openai-gpt-5-6-terra", "label": "openai/gpt-5.6-terra", "model": "openai/gpt-5.6-terra", "provider": "OpenAI", "harness": null, "effort": "max", "score": 59.588, "costUsd": 5.178734, "uncertainty": { "lower": 58.257, "upper": 60.919000000000004, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-5.6-terra" }, { "label": "Standard error", "value": "±1.33 points" }, { "label": "Mean latency", "value": "1,730 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "54.4%" }, { "label": "EMB", "value": "66.2%" }, { "label": "Terminal-Bench 2.1", "value": "77.5%" }, { "label": "Vibe Code Bench", "value": "74.6%" }, { "label": "Code Migration", "value": "48.2%" }, { "label": "Legal Research", "value": "41.3%" }, { "label": "HLAB", "value": "0.8%" } ] }, { "id": "openai-gpt-6-astra", "label": "openai/gpt-6-astra", "model": "openai/gpt-6-astra", "provider": "OpenAI", "harness": null, "effort": "max", "score": 66.608, "costUsd": 19.088776, "uncertainty": { "lower": 65.515, "upper": 67.70100000000001, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-6-astra" }, { "label": "Standard error", "value": "±1.09 points" }, { "label": "Mean latency", "value": "1,511 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "53.5%" }, { "label": "EMB", "value": "71.7%" }, { "label": "Terminal-Bench 2.1", "value": "87.3%" }, { "label": "Vibe Code Bench", "value": "89.6%" }, { "label": "Code Migration", "value": "68.5%" }, { "label": "Legal Research", "value": "39.4%" }, { "label": "HLAB", "value": "5.4%" } ] }, { "id": "openai-gpt-6-luna", "label": "openai/gpt-6-luna", "model": "openai/gpt-6-luna", "provider": "OpenAI", "harness": null, "effort": "max", "score": 58.45, "costUsd": 0.418873, "uncertainty": { "lower": 57.343, "upper": 59.557, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-6-luna" }, { "label": "Standard error", "value": "±1.11 points" }, { "label": "Mean latency", "value": "1,590 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "49.9%" }, { "label": "EMB", "value": "68.5%" }, { "label": "Terminal-Bench 2.1", "value": "73.0%" }, { "label": "Vibe Code Bench", "value": "81.6%" }, { "label": "Code Migration", "value": "44.4%" }, { "label": "Legal Research", "value": "30.3%" }, { "label": "HLAB", "value": "2.9%" } ] }, { "id": "openai-gpt-6-sol", "label": "openai/gpt-6-sol", "model": "openai/gpt-6-sol", "provider": "OpenAI", "harness": null, "effort": "max", "score": 62.569, "costUsd": 7.564103, "uncertainty": { "lower": 61.615, "upper": 63.523, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "openai/gpt-6-sol" }, { "label": "Standard error", "value": "±0.95 points" }, { "label": "Mean latency", "value": "1,594 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "49.0%" }, { "label": "EMB", "value": "71.5%" }, { "label": "Terminal-Bench 2.1", "value": "83.1%" }, { "label": "Vibe Code Bench", "value": "87.8%" }, { "label": "Code Migration", "value": "56.9%" }, { "label": "Legal Research", "value": "28.8%" }, { "label": "HLAB", "value": "1.7%" } ] }, { "id": "tencent-hy4-preview", "label": "tencent/hy4-preview", "model": "tencent/hy4-preview", "provider": "Tencent", "harness": null, "effort": null, "score": 55.402, "costUsd": 1.280885, "uncertainty": { "lower": 54.154, "upper": 56.65, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "tencent/hy4-preview" }, { "label": "Standard error", "value": "±1.25 points" }, { "label": "Mean latency", "value": "2,871 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "55.1%" }, { "label": "EMB", "value": "58.1%" }, { "label": "Terminal-Bench 2.1", "value": "55.1%" }, { "label": "Vibe Code Bench", "value": "77.5%" }, { "label": "Code Migration", "value": "46.8%" }, { "label": "Legal Research", "value": "45.2%" }, { "label": "HLAB", "value": "9.2%" } ] }, { "id": "thinkingmachines-inkling", "label": "thinkingmachines/inkling", "model": "thinkingmachines/inkling", "provider": "Thinkingmachines", "harness": null, "effort": "0.99", "score": 34.102, "costUsd": 1.354489, "uncertainty": { "lower": 32.97, "upper": 35.233999999999995, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "thinkingmachines/inkling" }, { "label": "Standard error", "value": "±1.13 points" }, { "label": "Mean latency", "value": "1,177 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "46.6%" }, { "label": "EMB", "value": "38.4%" }, { "label": "Terminal-Bench 2.1", "value": "47.6%" }, { "label": "Vibe Code Bench", "value": "19.2%" }, { "label": "Code Migration", "value": "11.8%" }, { "label": "Legal Research", "value": "28.4%" }, { "label": "HLAB", "value": "2.1%" } ] }, { "id": "thinkingmachines-inkling-small", "label": "thinkingmachines/inkling-small", "model": "thinkingmachines/inkling-small", "provider": "Thinkingmachines", "harness": null, "effort": "0.99", "score": 31.862, "costUsd": 0.281249, "uncertainty": { "lower": 30.758999999999997, "upper": 32.964999999999996, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "thinkingmachines/inkling-small" }, { "label": "Standard error", "value": "±1.10 points" }, { "label": "Mean latency", "value": "1,373 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "41.3%" }, { "label": "EMB", "value": "31.8%" }, { "label": "Terminal-Bench 2.1", "value": "55.1%" }, { "label": "Vibe Code Bench", "value": "19.1%" }, { "label": "Code Migration", "value": "13.3%" }, { "label": "Legal Research", "value": "25.5%" }, { "label": "HLAB", "value": "1.7%" } ] }, { "id": "xiaomi-mimo-v2-5", "label": "xiaomi/mimo-v2.5", "model": "xiaomi/mimo-v2.5", "provider": "Xiaomi", "harness": null, "effort": null, "score": 39.91, "costUsd": 0.061288, "uncertainty": { "lower": 38.598, "upper": 41.221999999999994, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.5" }, { "label": "Standard error", "value": "±1.31 points" }, { "label": "Mean latency", "value": "1,093 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "36.7%" }, { "label": "EMB", "value": "55.1%" }, { "label": "Terminal-Bench 2.1", "value": "60.7%" }, { "label": "Vibe Code Bench", "value": "42.2%" }, { "label": "Code Migration", "value": "13.4%" }, { "label": "Legal Research", "value": "9.1%" }, { "label": "HLAB", "value": "1.7%" } ] }, { "id": "xiaomi-mimo-v2-5-pro", "label": "xiaomi/mimo-v2.5-pro", "model": "xiaomi/mimo-v2.5-pro", "provider": "Xiaomi", "harness": null, "effort": null, "score": 40.974, "costUsd": 0.152141, "uncertainty": { "lower": 39.596999999999994, "upper": 42.351, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.5-pro" }, { "label": "Standard error", "value": "±1.38 points" }, { "label": "Mean latency", "value": "1,350 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "41.5%" }, { "label": "EMB", "value": "55.2%" }, { "label": "Terminal-Bench 2.1", "value": "57.3%" }, { "label": "Vibe Code Bench", "value": "34.1%" }, { "label": "Code Migration", "value": "20.4%" }, { "label": "Legal Research", "value": "15.9%" }, { "label": "HLAB", "value": "2.1%" } ] }, { "id": "xiaomi-mimo-v2-6-flash", "label": "xiaomi/mimo-v2.6-flash", "model": "xiaomi/mimo-v2.6-flash", "provider": "Xiaomi", "harness": null, "effort": null, "score": 59.584, "costUsd": 0.19671, "uncertainty": { "lower": 58.451, "upper": 60.717000000000006, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.6-flash" }, { "label": "Standard error", "value": "±1.13 points" }, { "label": "Mean latency", "value": "2,702 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "56.3%" }, { "label": "EMB", "value": "65.5%" }, { "label": "Terminal-Bench 2.1", "value": "76.4%" }, { "label": "Vibe Code Bench", "value": "79.0%" }, { "label": "Code Migration", "value": "40.4%" }, { "label": "Legal Research", "value": "38.0%" }, { "label": "HLAB", "value": "11.3%" } ] }, { "id": "xiaomi-mimo-v2-6-pro", "label": "xiaomi/mimo-v2.6-pro", "model": "xiaomi/mimo-v2.6-pro", "provider": "Xiaomi", "harness": null, "effort": null, "score": 59.47, "costUsd": 0.38918, "uncertainty": { "lower": 58.293, "upper": 60.647, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "xiaomi/mimo-v2.6-pro" }, { "label": "Standard error", "value": "±1.18 points" }, { "label": "Mean latency", "value": "3,140 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "57.3%" }, { "label": "EMB", "value": "62.9%" }, { "label": "Terminal-Bench 2.1", "value": "67.8%" }, { "label": "Vibe Code Bench", "value": "85.2%" }, { "label": "Code Migration", "value": "42.3%" }, { "label": "Legal Research", "value": "47.1%" }, { "label": "HLAB", "value": "10.8%" } ] }, { "id": "zai-glm-5-2", "label": "zai/glm-5.2", "model": "zai/glm-5.2", "provider": "Zhipu AI", "harness": null, "effort": null, "score": 53.122, "costUsd": 4.290631, "uncertainty": { "lower": 51.938, "upper": 54.306, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "zai/glm-5.2" }, { "label": "Standard error", "value": "±1.18 points" }, { "label": "Mean latency", "value": "2,527 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "49.7%" }, { "label": "EMB", "value": "61.5%" }, { "label": "Terminal-Bench 2.1", "value": "67.8%" }, { "label": "Vibe Code Bench", "value": "64.0%" }, { "label": "Code Migration", "value": "38.8%" }, { "label": "Legal Research", "value": "31.3%" }, { "label": "HLAB", "value": "7.1%" } ] }, { "id": "zai-glm-5-3", "label": "zai/glm-5.3", "model": "zai/glm-5.3", "provider": "Fireworks AI", "harness": null, "effort": "max", "score": 56.97, "costUsd": 6.73155, "uncertainty": { "lower": 55.621, "upper": 58.318999999999996, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "zai/glm-5.3" }, { "label": "Standard error", "value": "±1.35 points" }, { "label": "Mean latency", "value": "3,989 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "55.8%" }, { "label": "EMB", "value": "56.3%" }, { "label": "Terminal-Bench 2.1", "value": "71.5%" }, { "label": "Vibe Code Bench", "value": "78.1%" }, { "label": "Code Migration", "value": "43.2%" }, { "label": "Legal Research", "value": "49.0%" }, { "label": "HLAB", "value": "8.3%" } ] }, { "id": "zai-glm-5-3-flash", "label": "zai/glm-5.3-flash", "model": "zai/glm-5.3-flash", "provider": "Fireworks AI", "harness": null, "effort": "max", "score": 47.216, "costUsd": 0.992509, "uncertainty": { "lower": 45.771, "upper": 48.661, "label": "±1 standard error" }, "sourceUrl": "https://www.vals.ai/benchmarks/vals_index", "details": [ { "label": "Publisher model id", "value": "zai/glm-5.3-flash" }, { "label": "Standard error", "value": "±1.45 points" }, { "label": "Mean latency", "value": "4,267 seconds per test" }, { "label": "Evaluation mode", "value": "Agentic · many steps allowed" }, { "label": "Finance Agent v2", "value": "57.9%" }, { "label": "EMB", "value": "55.9%" }, { "label": "Terminal-Bench 2.1", "value": "62.9%" }, { "label": "Vibe Code Bench", "value": "30.8%" }, { "label": "Code Migration", "value": "20.2%" }, { "label": "Legal Research", "value": "45.2%" }, { "label": "HLAB", "value": "6.7%" } ] } ] } }