Differentiate audience modes and tighten eval navigation

app/evals/page.tsx

@@ -2,13 +2,14 @@
 
 import { useCallback, useEffect, useMemo, useRef, useState } from "react"
 import { useRouter } from "next/navigation"
-import { ArrowLeft, Search, X } from "lucide-react"
+import { ArrowLeft, ChevronDown, Search, SlidersHorizontal, X } from "lucide-react"
 
 import { useAudienceMode } from "@/components/audience-mode-provider"
 import { ListPagination } from "@/components/list-pagination"
 import { Navigation } from "@/components/navigation"
 import { PageHeader } from "@/components/page-header"
 import { Button } from "@/components/ui/button"
+import { Collapsible, CollapsibleContent, CollapsibleTrigger } from "@/components/ui/collapsible"
 import { Input } from "@/components/ui/input"
 import type { EvalHierarchy } from "@/lib/backend-artifacts"
 import type { BenchmarkCard, CategoryType } from "@/lib/benchmark-schema"
@@ -262,6 +263,7 @@ interface EvalBrowserNode {
   domains: string[]
   dataType?: string
   license?: string
+  card?: BenchmarkCard
   modelsCount: number
   metricCount: number
   topScore?: number
@@ -353,6 +355,71 @@ function formatCompactScore(value: number | undefined) {
   return value.toFixed(value >= 100 ? 0 : 2)
 }
 
+function hasConcreteText(value: string | undefined | null) {
+  if (!value) return false
+  const normalized = value.trim().toLowerCase()
+  return Boolean(normalized) && normalized !== "not specified" && normalized !== "unknown"
+}
+
+function firstConcreteListValue(value: string[] | string | undefined | null) {
+  if (Array.isArray(value)) {
+    return value.find((entry) => hasConcreteText(entry))
+  }
+
+  return hasConcreteText(value) ? value : undefined
+}
+
+function countConcreteListValues(value: string[] | undefined | null) {
+  return (value ?? []).filter((entry) => hasConcreteText(entry)).length
+}
+
+function getNodePolicySummary(node: EvalBrowserNode) {
+  const card = node.card
+  if (!card) return null
+
+  const riskCount = card.possible_risks?.length ?? 0
+  const reportingGapCount =
+    card.missing_fields?.filter(
+      (field) => field.startsWith("methodology") || field.startsWith("purpose_and_intended_users")
+    ).length ?? 0
+
+  return {
+    goal: hasConcreteText(card.purpose_and_intended_users?.goal)
+      ? card.purpose_and_intended_users.goal
+      : undefined,
+    limitations: hasConcreteText(card.purpose_and_intended_users?.limitations)
+      ? card.purpose_and_intended_users.limitations
+      : undefined,
+    audience: firstConcreteListValue(card.purpose_and_intended_users?.audience),
+    compliance: hasConcreteText(card.ethical_and_legal_considerations?.compliance_with_regulations)
+      ? card.ethical_and_legal_considerations.compliance_with_regulations
+      : undefined,
+    riskCount,
+    reportingGapCount,
+  }
+}
+
+function getNodeResearchSummary(node: EvalBrowserNode) {
+  const card = node.card
+  if (!card) return null
+
+  const similarBenchmarks = Array.isArray(card.benchmark_details?.similar_benchmarks)
+    ? card.benchmark_details.similar_benchmarks
+    : card.benchmark_details?.similar_benchmarks
+      ? [card.benchmark_details.similar_benchmarks]
+      : []
+
+  return {
+    methodsCount: countConcreteListValues(card.methodology?.methods),
+    metricsCount: countConcreteListValues(card.methodology?.metrics),
+    similarCount: countConcreteListValues(similarBenchmarks),
+    interpretation: hasConcreteText(card.methodology?.interpretation)
+      ? card.methodology.interpretation
+      : undefined,
+    missingMethodCount: card.missing_fields?.filter((field) => field.startsWith("methodology")).length ?? 0,
+  }
+}
+
 function getNodeCard(
   benchmarkCards: Record<string, BenchmarkCard>,
   ...candidates: Array<string | undefined>
@@ -452,6 +519,7 @@ function mapHierarchyCategory(value: string | undefined | null): CategoryType {
 export default function EvalsPage() {
   const { mode } = useAudienceMode()
   const router = useRouter()
+  const isResearchView = mode === "research"
 
   const [summaries, setSummaries] = useState<BenchmarkEvalListItem[]>([])
   const [benchmarkCards, setBenchmarkCards] = useState<Record<string, BenchmarkCard>>({})
@@ -464,6 +532,7 @@ export default function EvalsPage() {
   const [selectedNodeKind, setSelectedNodeKind] = useState<EvalBrowserNodeKind | null>(null)
   const [currentNodeId, setCurrentNodeId] = useState<string | null>(null)
   const [page, setPage] = useState(1)
+  const [filtersOpen, setFiltersOpen] = useState(false)
   const pendingHistoryActionRef = useRef<"push" | "replace">("replace")
 
   useEffect(() => {
@@ -635,6 +704,7 @@ export default function EvalsPage() {
         domains: Array.from(new Set(domains.flatMap((domain) => normalizeDomainList(domain)))),
         dataType: card?.benchmark_details?.data_type,
         license: card?.ethical_and_legal_considerations?.data_licensing,
+        card,
         modelsCount: stats.modelsCount,
         metricCount: stats.metricCount,
         topScore: stats.topScore,
@@ -655,9 +725,9 @@ export default function EvalsPage() {
         metrics?: Array<{ key: string; display_name: string }>
       }>,
       scopeKeys: string[]
-    ): EvalBrowserNode["matrixPreview"] | null => {
+    ): EvalBrowserNode["matrixPreview"] | undefined => {
       if (benchmarks.length < 2) {
-        return null
+        return undefined
       }
 
       const metricLabels = new Set<string>()
@@ -665,7 +735,7 @@ export default function EvalsPage() {
 
       for (const benchmark of benchmarks) {
         if ((benchmark.slices?.length ?? 0) > 0 || (benchmark.metrics?.length ?? 0) !== 1) {
-          return null
+          return undefined
         }
 
         const metric = benchmark.metrics?.[0]
@@ -679,7 +749,7 @@ export default function EvalsPage() {
       }
 
       if (metricLabels.size !== 1) {
-        return null
+        return undefined
       }
 
       return {
@@ -747,6 +817,10 @@ export default function EvalsPage() {
       const card = summary?.benchmark_card ?? getNodeCard(benchmarkCards, ...cardCandidates)
       const childSlices = slices.filter((slice) => !isSameHierarchyKey(slice.key, benchmarkKey))
       const drilldownSlices = childSlices.filter((slice) => (slice.metrics?.length ?? 0) > 1)
+      const fallbackSummary =
+        !summary && metrics.length > 0
+          ? scopeKeys.map((scopeKey) => pickSummaryForKey(summariesWithCards, scopeKey, scopeKeys)).find(Boolean)
+          : undefined
       const isParentRollupBenchmark =
         Boolean(parentId) && scopeKeys.some((scopeKey) => isSameHierarchyKey(scopeKey, benchmarkKey))
 
@@ -775,9 +849,14 @@ export default function EvalsPage() {
         domains,
         summaries: summary ? [summary] : [],
         card,
-        href: drilldownSlices.length === 0 && summary
-          ? `/evals/${summary.evaluation_id}`
-          : undefined,
+        href:
+          drilldownSlices.length === 0
+            ? summary
+              ? `/evals/${summary.evaluation_id}`
+              : fallbackSummary
+                ? `/evals/${fallbackSummary.evaluation_id}`
+                : undefined
+            : undefined,
         scopeKeys,
         descriptionFallback: `Browse the {label} benchmark and its lower-level breakdowns.`,
       })
@@ -874,6 +953,8 @@ export default function EvalsPage() {
           const suiteBenchmarks = (composite.benchmarks ?? []).filter((benchmark) => !isSameHierarchyKey(benchmark.key, composite.key))
           const suiteMatrixPreview = buildSingleMetricMatrixPreview(suiteBenchmarks, suiteScopeKeys)
           const rollupSummary = pickSummaryForKey(summariesWithCards, composite.key, suiteScopeKeys)
+          const hasSuiteRollup = Boolean(rollupBenchmark && rollupSummary)
+          const syntheticMatrixEvalId = suiteMatrixPreview && !hasSuiteRollup ? `matrix__${composite.key}` : undefined
 
           buildNode({
             id: suiteId,
@@ -886,7 +967,13 @@ export default function EvalsPage() {
             summaries: suiteSummaries,
             card: suiteCard,
             sourceLabel: suiteLabel,
-            href: suiteMatrixPreview && rollupSummary ? `/evals/${rollupSummary.evaluation_id}` : undefined,
+            href: suiteMatrixPreview
+              ? hasSuiteRollup
+                ? `/evals/${rollupSummary.evaluation_id}`
+                : syntheticMatrixEvalId
+                  ? `/evals/${syntheticMatrixEvalId}`
+                  : undefined
+              : undefined,
             scopeKeys: suiteScopeKeys,
             matrixPreview: suiteMatrixPreview,
             descriptionFallback: `Browse the {label} suite and then open its benchmark children.`,
@@ -928,8 +1015,12 @@ export default function EvalsPage() {
           }
 
           const suiteNode = nodes.get(suiteId)
-          if (suiteNode && suiteNode.childIds.length === 0 && rollupSummary) {
-            suiteNode.href = `/evals/${rollupSummary.evaluation_id}`
+          if (suiteNode && suiteNode.childIds.length === 0) {
+            if (hasSuiteRollup) {
+              suiteNode.href = `/evals/${rollupSummary.evaluation_id}`
+            } else if (syntheticMatrixEvalId) {
+              suiteNode.href = `/evals/${syntheticMatrixEvalId}`
+            }
           }
         }
 
@@ -979,6 +1070,8 @@ export default function EvalsPage() {
         const visibleBenchmarks = suiteBenchmarks.filter((benchmark) => !isSameHierarchyKey(benchmark.key, suiteKey))
         const suiteMatrixPreview = buildSingleMetricMatrixPreview(visibleBenchmarks, suiteScopeKeys)
         const rollupSummary = pickSummaryForKey(summariesWithCards, suiteKey, suiteScopeKeys)
+        const hasSuiteRollup = Boolean(rollupBenchmark && rollupSummary)
+        const syntheticMatrixEvalId = suiteMatrixPreview && !hasSuiteRollup ? `matrix__${suiteKey}` : undefined
         const suiteSummaries = summariesWithCards.filter((summary) => {
           const familyScope = getSummaryScopeKey(summary.benchmark_family_key ?? summary.composite_benchmark_key)
           return familyScope === getSummaryScopeKey(suiteKey)
@@ -1002,7 +1095,13 @@ export default function EvalsPage() {
             family.key
           ),
           sourceLabel: suiteLabel,
-          href: suiteMatrixPreview && rollupSummary ? `/evals/${rollupSummary.evaluation_id}` : undefined,
+          href: suiteMatrixPreview
+            ? hasSuiteRollup
+              ? `/evals/${rollupSummary.evaluation_id}`
+              : syntheticMatrixEvalId
+                ? `/evals/${syntheticMatrixEvalId}`
+                : undefined
+            : undefined,
           scopeKeys: suiteScopeKeys,
           matrixPreview: suiteMatrixPreview,
           descriptionFallback: `Browse the {label} suite and then open its benchmark children.`,
@@ -1044,8 +1143,12 @@ export default function EvalsPage() {
         }
 
         const suiteNode = nodes.get(suiteId)
-        if (suiteNode && suiteNode.childIds.length === 0 && rollupSummary) {
-          suiteNode.href = `/evals/${rollupSummary.evaluation_id}`
+        if (suiteNode && suiteNode.childIds.length === 0) {
+          if (hasSuiteRollup) {
+            suiteNode.href = `/evals/${rollupSummary.evaluation_id}`
+          } else if (syntheticMatrixEvalId) {
+            suiteNode.href = `/evals/${syntheticMatrixEvalId}`
+          }
         }
 
         continue
@@ -1250,6 +1353,12 @@ export default function EvalsPage() {
     }
   }, [currentLevelKinds, selectedNodeKind])
 
+  useEffect(() => {
+    if (activeFilterCount > 0) {
+      setFiltersOpen(true)
+    }
+  }, [activeFilterCount])
+
   const handleNodeOpen = useCallback(
     (node: EvalBrowserNode) => {
       if (node.childIds.length > 0) {
@@ -1455,116 +1564,150 @@ export default function EvalsPage() {
             </div>
           </div>
 
-          {currentLevelKinds.length > 1 && (
-            <div className="mt-4 space-y-1.5">
-              <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-stone-500 dark:text-stone-400">
-                Granularity
-              </div>
-              <div className="flex flex-wrap items-center gap-1.5">
-                <button
-                  type="button"
-                  onClick={() => setSelectedNodeKind(null)}
-                  className={cn(
-                    "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
-                    selectedNodeKind === null
-                      ? "border-stone-950 bg-stone-950 text-stone-50 dark:border-stone-100 dark:bg-stone-100 dark:text-stone-950"
-                      : "border-stone-200/80 bg-stone-50/80 text-stone-600 hover:bg-stone-100 dark:border-stone-700/80 dark:bg-stone-900/70 dark:text-stone-300 dark:hover:bg-stone-800"
+          <Collapsible
+            open={filtersOpen}
+            onOpenChange={setFiltersOpen}
+            className="mt-4 rounded-[1.35rem] border border-stone-200/80 bg-white/70 dark:border-stone-800/80 dark:bg-stone-950/60"
+          >
+            <CollapsibleTrigger asChild>
+              <button type="button" className="flex w-full items-center justify-between gap-3 px-4 py-3 text-left">
+                <div className="min-w-0">
+                  <div className="flex items-center gap-2 text-[11px] font-semibold uppercase tracking-[0.2em] text-stone-500 dark:text-stone-400">
+                    <SlidersHorizontal className="h-3.5 w-3.5" />
+                    Refine this list
+                  </div>
+                  <p className="mt-1 text-sm text-stone-600 dark:text-stone-300">
+                    {activeFilterCount > 0
+                      ? `${activeFilterCount} filter${activeFilterCount === 1 ? "" : "s"} active`
+                      : "Open filters only when you need to narrow by node type, domain tags, or category."}
+                  </p>
+                </div>
+                <div className="flex items-center gap-2">
+                  {activeFilterCount > 0 && (
+                    <span className="rounded-full border border-stone-200/80 bg-stone-100/80 px-2.5 py-1 text-[11px] font-medium text-stone-700 dark:border-stone-700/80 dark:bg-stone-900/80 dark:text-stone-200">
+                      {activeFilterCount} active
+                    </span>
                   )}
-                >
-                  All
-                </button>
-                {currentLevelKinds.map((kind) => (
-                  <button
-                    key={kind}
-                    type="button"
-                    onClick={() => setSelectedNodeKind(selectedNodeKind === kind ? null : kind)}
-                    className={cn(
-                      "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
-                      selectedNodeKind === kind
-                        ? "border-sky-300 bg-sky-50 text-sky-800 dark:border-sky-800 dark:bg-sky-950/50 dark:text-sky-200"
-                        : "border-stone-200/80 bg-white text-stone-600 hover:bg-stone-50 dark:border-stone-700/80 dark:bg-stone-900 dark:text-stone-300 dark:hover:bg-stone-800"
-                    )}
-                  >
-                    {getBrowserNodeKindLabel(kind)}
-                  </button>
-                ))}
-              </div>
-            </div>
-          )}
+                  <ChevronDown className={cn("h-4 w-4 text-stone-500 transition-transform dark:text-stone-400", filtersOpen && "rotate-180")} />
+                </div>
+              </button>
+            </CollapsibleTrigger>
+
+            <CollapsibleContent>
+              <div className="border-t border-stone-200/80 px-4 pb-4 pt-4 dark:border-stone-800/80">
+                {currentLevelKinds.length > 1 && (
+                  <div className="space-y-1.5">
+                    <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-stone-500 dark:text-stone-400">
+                      Node type
+                    </div>
+                    <div className="flex flex-wrap items-center gap-1.5">
+                      <button
+                        type="button"
+                        onClick={() => setSelectedNodeKind(null)}
+                        className={cn(
+                          "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
+                          selectedNodeKind === null
+                            ? "border-stone-950 bg-stone-950 text-stone-50 dark:border-stone-100 dark:bg-stone-100 dark:text-stone-950"
+                            : "border-stone-200/80 bg-stone-50/80 text-stone-600 hover:bg-stone-100 dark:border-stone-700/80 dark:bg-stone-900/70 dark:text-stone-300 dark:hover:bg-stone-800"
+                        )}
+                      >
+                        All
+                      </button>
+                      {currentLevelKinds.map((kind) => (
+                        <button
+                          key={kind}
+                          type="button"
+                          onClick={() => setSelectedNodeKind(selectedNodeKind === kind ? null : kind)}
+                          className={cn(
+                            "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
+                            selectedNodeKind === kind
+                              ? "border-sky-300 bg-sky-50 text-sky-800 dark:border-sky-800 dark:bg-sky-950/50 dark:text-sky-200"
+                              : "border-stone-200/80 bg-white text-stone-600 hover:bg-stone-50 dark:border-stone-700/80 dark:bg-stone-900 dark:text-stone-300 dark:hover:bg-stone-800"
+                          )}
+                        >
+                          {getBrowserNodeKindLabel(kind)}
+                        </button>
+                      ))}
+                    </div>
+                  </div>
+                )}
 
-          {allDomains.length > 0 && (
-            <div className="mt-4 space-y-1.5">
-              <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-stone-500 dark:text-stone-400">
-                Domain
-              </div>
-              <div className="flex flex-wrap items-center gap-1.5">
-                  <button
-                    type="button"
-                    onClick={() => setSelectedDomain(null)}
-                    className={cn(
-                      "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
-                      selectedDomain === null
-                        ? "border-stone-950 bg-stone-950 text-stone-50 dark:border-stone-100 dark:bg-stone-100 dark:text-stone-950"
-                        : "border-stone-200/80 bg-stone-50/80 text-stone-600 hover:bg-stone-100 dark:border-stone-700/80 dark:bg-stone-900/70 dark:text-stone-300 dark:hover:bg-stone-800"
-                    )}
-                  >
-                    All
-                  </button>
-                  {allDomains.map((domain) => (
-                    <button
-                      key={domain}
-                      type="button"
-                      onClick={() => setSelectedDomain(selectedDomain === domain ? null : domain)}
-                      className={cn(
-                        "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
-                        selectedDomain === domain
-                          ? "border-sky-300 bg-sky-50 text-sky-800 dark:border-sky-800 dark:bg-sky-950/50 dark:text-sky-200"
-                          : "border-stone-200/80 bg-white text-stone-600 hover:bg-stone-50 dark:border-stone-700/80 dark:bg-stone-900 dark:text-stone-300 dark:hover:bg-stone-800"
-                      )}
-                    >
-                      {domain}
-                    </button>
-                  ))}
-              </div>
-            </div>
-          )}
+                {allDomains.length > 0 && (
+                  <div className="mt-4 space-y-1.5">
+                    <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-stone-500 dark:text-stone-400">
+                      Domain tags
+                    </div>
+                    <div className="flex max-h-40 flex-wrap items-center gap-1.5 overflow-y-auto pr-1">
+                      <button
+                        type="button"
+                        onClick={() => setSelectedDomain(null)}
+                        className={cn(
+                          "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
+                          selectedDomain === null
+                            ? "border-stone-950 bg-stone-950 text-stone-50 dark:border-stone-100 dark:bg-stone-100 dark:text-stone-950"
+                            : "border-stone-200/80 bg-stone-50/80 text-stone-600 hover:bg-stone-100 dark:border-stone-700/80 dark:bg-stone-900/70 dark:text-stone-300 dark:hover:bg-stone-800"
+                        )}
+                      >
+                        All
+                      </button>
+                      {allDomains.map((domain) => (
+                        <button
+                          key={domain}
+                          type="button"
+                          onClick={() => setSelectedDomain(selectedDomain === domain ? null : domain)}
+                          className={cn(
+                            "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
+                            selectedDomain === domain
+                              ? "border-sky-300 bg-sky-50 text-sky-800 dark:border-sky-800 dark:bg-sky-950/50 dark:text-sky-200"
+                              : "border-stone-200/80 bg-white text-stone-600 hover:bg-stone-50 dark:border-stone-700/80 dark:bg-stone-900 dark:text-stone-300 dark:hover:bg-stone-800"
+                          )}
+                        >
+                          {domain}
+                        </button>
+                      ))}
+                    </div>
+                  </div>
+                )}
 
-          {allCategories.length > 0 && (
-            <div className="mt-4 space-y-1.5">
-              <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-stone-500 dark:text-stone-400">
-                Category
-              </div>
-              <div className="flex flex-wrap items-center gap-1.5">
-                  <button
-                    type="button"
-                    onClick={() => setSelectedCategory(null)}
-                    className={cn(
-                      "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
-                      selectedCategory === null
-                        ? "border-stone-950 bg-stone-950 text-stone-50 dark:border-stone-100 dark:bg-stone-100 dark:text-stone-950"
-                        : "border-stone-200/80 bg-stone-50/80 text-stone-600 hover:bg-stone-100 dark:border-stone-700/80 dark:bg-stone-900/70 dark:text-stone-300 dark:hover:bg-stone-800"
-                    )}
-                  >
-                    All
-                  </button>
-                  {allCategories.map((category) => (
-                    <button
-                      key={category}
-                      type="button"
-                      onClick={() => setSelectedCategory(selectedCategory === category ? null : category)}
-                      className={cn(
-                        "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
-                        selectedCategory === category
-                          ? `${getCategoryColor(category as CategoryType)} border`
-                          : "border-stone-200/80 bg-white text-stone-600 hover:bg-stone-50 dark:border-stone-700/80 dark:bg-stone-900 dark:text-stone-300 dark:hover:bg-stone-800"
-                      )}
-                    >
-                      {category}
-                    </button>
-                  ))}
+                {allCategories.length > 0 && (
+                  <div className="mt-4 space-y-1.5">
+                    <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-stone-500 dark:text-stone-400">
+                      Category
+                    </div>
+                    <div className="flex flex-wrap items-center gap-1.5">
+                      <button
+                        type="button"
+                        onClick={() => setSelectedCategory(null)}
+                        className={cn(
+                          "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
+                          selectedCategory === null
+                            ? "border-stone-950 bg-stone-950 text-stone-50 dark:border-stone-100 dark:bg-stone-100 dark:text-stone-950"
+                            : "border-stone-200/80 bg-stone-50/80 text-stone-600 hover:bg-stone-100 dark:border-stone-700/80 dark:bg-stone-900/70 dark:text-stone-300 dark:hover:bg-stone-800"
+                        )}
+                      >
+                        All
+                      </button>
+                      {allCategories.map((category) => (
+                        <button
+                          key={category}
+                          type="button"
+                          onClick={() => setSelectedCategory(selectedCategory === category ? null : category)}
+                          className={cn(
+                            "shrink-0 rounded-full border px-3 py-1.5 text-xs font-medium transition-colors",
+                            selectedCategory === category
+                              ? `${getCategoryColor(category as CategoryType)} border`
+                              : "border-stone-200/80 bg-white text-stone-600 hover:bg-stone-50 dark:border-stone-700/80 dark:bg-stone-900 dark:text-stone-300 dark:hover:bg-stone-800"
+                          )}
+                        >
+                          {category}
+                        </button>
+                      ))}
+                    </div>
+                  </div>
+                )}
               </div>
-            </div>
-          )}
+            </CollapsibleContent>
+          </Collapsible>
         </section>
 
         {filtered.length === 0 ? (
@@ -1580,6 +1723,8 @@ export default function EvalsPage() {
               const isNavigable = node.childIds.length > 0 || Boolean(node.href)
               const actionLabel = node.childIds.length > 0 ? "Open level" : node.matrixPreview ? "View rollup" : "View benchmark"
               const kindLabel = getBrowserNodeKindLabel(node.kind)
+              const policySummary = getNodePolicySummary(node)
+              const researchSummary = getNodeResearchSummary(node)
 
               return (
                 <button
@@ -1655,6 +1800,84 @@ export default function EvalsPage() {
                     </p>
                   )}
 
+                  {!isResearchView && policySummary && (
+                    <div className="mb-4 space-y-2 rounded-[1.1rem] border border-amber-200/80 bg-amber-50/70 p-3 dark:border-amber-900/50 dark:bg-amber-950/20">
+                      <div className="text-[10px] font-semibold uppercase tracking-[0.18em] text-amber-800 dark:text-amber-200">
+                        Policy notes
+                      </div>
+                      {policySummary.goal && (
+                        <p className="text-sm leading-6 text-stone-700 dark:text-stone-200">
+                          <span className="font-semibold">What it measures: </span>
+                          {policySummary.goal}
+                        </p>
+                      )}
+                      {policySummary.limitations && (
+                        <p className="text-sm leading-6 text-stone-700 dark:text-stone-200">
+                          <span className="font-semibold">Main caveat: </span>
+                          {policySummary.limitations}
+                        </p>
+                      )}
+                      <div className="flex flex-wrap gap-2 text-[11px] text-stone-700 dark:text-stone-200">
+                        {policySummary.audience && (
+                          <span className="rounded-full border border-amber-200/80 bg-white/80 px-2.5 py-1 dark:border-amber-900/50 dark:bg-stone-950/70">
+                            Intended for {policySummary.audience}
+                          </span>
+                        )}
+                        {policySummary.compliance && (
+                          <span className="rounded-full border border-amber-200/80 bg-white/80 px-2.5 py-1 dark:border-amber-900/50 dark:bg-stone-950/70">
+                            Regulation note documented
+                          </span>
+                        )}
+                        {policySummary.riskCount > 0 && (
+                          <span className="rounded-full border border-amber-200/80 bg-white/80 px-2.5 py-1 dark:border-amber-900/50 dark:bg-stone-950/70">
+                            {policySummary.riskCount} risk note{policySummary.riskCount === 1 ? "" : "s"}
+                          </span>
+                        )}
+                        {policySummary.reportingGapCount > 0 && (
+                          <span className="rounded-full border border-rose-200/80 bg-white/80 px-2.5 py-1 text-rose-700 dark:border-rose-900/50 dark:bg-stone-950/70 dark:text-rose-200">
+                            {policySummary.reportingGapCount} missing reporting field{policySummary.reportingGapCount === 1 ? "" : "s"}
+                          </span>
+                        )}
+                      </div>
+                    </div>
+                  )}
+
+                  {isResearchView && researchSummary && (
+                    <div className="mb-4 space-y-2 rounded-[1.1rem] border border-sky-200/80 bg-sky-50/70 p-3 dark:border-sky-900/50 dark:bg-sky-950/20">
+                      <div className="text-[10px] font-semibold uppercase tracking-[0.18em] text-sky-800 dark:text-sky-200">
+                        Research notes
+                      </div>
+                      {researchSummary.interpretation && (
+                        <p className="text-sm leading-6 text-stone-700 dark:text-stone-200">
+                          <span className="font-semibold">Score interpretation: </span>
+                          {researchSummary.interpretation}
+                        </p>
+                      )}
+                      <div className="flex flex-wrap gap-2 text-[11px] text-stone-700 dark:text-stone-200">
+                        {researchSummary.methodsCount > 0 && (
+                          <span className="rounded-full border border-sky-200/80 bg-white/80 px-2.5 py-1 dark:border-sky-900/50 dark:bg-stone-950/70">
+                            {researchSummary.methodsCount} method note{researchSummary.methodsCount === 1 ? "" : "s"}
+                          </span>
+                        )}
+                        {researchSummary.metricsCount > 0 && (
+                          <span className="rounded-full border border-sky-200/80 bg-white/80 px-2.5 py-1 dark:border-sky-900/50 dark:bg-stone-950/70">
+                            {researchSummary.metricsCount} documented metric{researchSummary.metricsCount === 1 ? "" : "s"}
+                          </span>
+                        )}
+                        {researchSummary.similarCount > 0 && (
+                          <span className="rounded-full border border-sky-200/80 bg-white/80 px-2.5 py-1 dark:border-sky-900/50 dark:bg-stone-950/70">
+                            {researchSummary.similarCount} related benchmark{researchSummary.similarCount === 1 ? "" : "s"}
+                          </span>
+                        )}
+                        {researchSummary.missingMethodCount > 0 && (
+                          <span className="rounded-full border border-rose-200/80 bg-white/80 px-2.5 py-1 text-rose-700 dark:border-rose-900/50 dark:bg-stone-950/70 dark:text-rose-200">
+                            {researchSummary.missingMethodCount} missing method field{researchSummary.missingMethodCount === 1 ? "" : "s"}
+                          </span>
+                        )}
+                      </div>
+                    </div>
+                  )}
+
                   {node.matrixPreview && (
                     <div className="mb-4 overflow-hidden rounded-[1.2rem] border border-stone-200/80 bg-stone-50/85 dark:border-stone-800/80 dark:bg-stone-900/85">
                       <div className="space-y-2 bg-white/92 px-3 py-3 dark:bg-stone-950/92">
@@ -1679,18 +1902,18 @@ export default function EvalsPage() {
                   <div className="mb-4 grid gap-px overflow-hidden rounded-[1.2rem] border border-stone-200/80 bg-stone-200/80 dark:border-stone-800/80 dark:bg-stone-800/80 sm:grid-cols-2 xl:grid-cols-3">
                     {topScoreLabel !== "—" && (
                       <div className="bg-white/90 px-3 py-3 dark:bg-stone-950/90">
-                        <div className="text-[10px] font-semibold uppercase tracking-[0.16em] text-stone-500 dark:text-stone-400">Top score</div>
+                        <div className="text-[10px] font-semibold uppercase tracking-[0.16em] text-stone-500 dark:text-stone-400">Reported score</div>
                         <div className="mt-1 text-sm font-semibold text-stone-900 dark:text-stone-100">{topScoreLabel}</div>
                       </div>
                     )}
                     <div className="bg-white/90 px-3 py-3 dark:bg-stone-950/90">
-                      <div className="text-[10px] font-semibold uppercase tracking-[0.16em] text-stone-500 dark:text-stone-400">Source</div>
+                      <div className="text-[10px] font-semibold uppercase tracking-[0.16em] text-stone-500 dark:text-stone-400">Dataset or record</div>
                       <div className="mt-1 truncate text-sm font-semibold text-stone-900 dark:text-stone-100">
                         {node.sourceLabel}
                       </div>
                     </div>
                     <div className="bg-white/90 px-3 py-3 dark:bg-stone-950/90">
-                      <div className="text-[10px] font-semibold uppercase tracking-[0.16em] text-stone-500 dark:text-stone-400">Instance data</div>
+                      <div className="text-[10px] font-semibold uppercase tracking-[0.16em] text-stone-500 dark:text-stone-400">Linked instances</div>
                       <div className="mt-1 text-sm font-semibold text-stone-900 dark:text-stone-100">
                         {node.instanceDataLabel}
                       </div>

app/page.tsx

@@ -57,7 +57,7 @@ export default async function HomePage() {
                   Public reporting for AI evaluations.
                 </h1>
                 <p className="max-w-3xl text-lg leading-8 text-muted-foreground sm:text-xl">
-                  Start from the question you have: which models show broad evidence, which benchmarks have thin reporting, and where the public record is still incomplete.
+                  Start from the question you have: which models have more reported benchmarks, which benchmarks are sparsely reported, and where the record is still incomplete.
                 </p>
               </div>
 
@@ -86,20 +86,20 @@ export default async function HomePage() {
                 <div className="grid md:grid-cols-3 md:divide-x md:divide-border/60">
                   <SignalCard
                     icon={<Database className="h-4 w-4" />}
-                    title="Coverage"
-                    body="Benchmark breadth, setup details, and provenance stay visible."
+                    title="Reported benchmarks"
+                    body="See which benchmarks, settings, and sources are actually documented."
                     tone="sky"
                   />
                   <SignalCard
                     icon={<Scale className="h-4 w-4" />}
-                    title="Comparability"
-                    body="Configuration gaps and evaluator relationships remain part of the reading."
+                    title="Comparison context"
+                    body="Configuration gaps and evaluator relationships stay attached to each record."
                     tone="amber"
                   />
                   <SignalCard
                     icon={<BookOpenText className="h-4 w-4" />}
-                    title="Reading modes"
-                    body="Research and policy views shift emphasis without hiding the record."
+                    title="Reader modes"
+                    body="Research and policy views prioritize different fields without hiding the same record."
                     tone="emerald"
                   />
                 </div>
@@ -113,7 +113,7 @@ export default async function HomePage() {
                     Start from a reader question
                   </div>
                   <div className="mt-1 text-sm leading-6 text-muted-foreground">
-                    The site is strongest when you treat it as an evidence reader, not a leaderboard.
+                    The site works best when you inspect reported records, not just the ranking order.
                   </div>
                 </div>
                 <Database className="h-4 w-4 text-muted-foreground" />
@@ -121,27 +121,27 @@ export default async function HomePage() {
 
               <div className="mt-5 space-y-3">
                 <InquiryRow
-                  title="Which models have broad public evidence?"
-                  body="Use the models view to scan benchmark breadth, reported results, and where coverage is still thin."
+                  title="Which models have the most reported benchmarks?"
+                  body="Use the models view to scan reported benchmarks, result counts, and where reporting is sparse."
                   href="/models"
                 />
                 <InquiryRow
                   title="How is one benchmark being reported?"
-                  body="Use the evaluations view to inspect methodology context, score spread, and missing configuration details."
+                  body="Use the evaluations view to inspect benchmark notes, score ranges, and missing setup details."
                   href="/evals"
                 />
                 <InquiryRow
-                  title="Who is publishing the evidence?"
-                  body="Group models by developer or open a developer page to see which organizations are reporting the most."
+                  title="Which organizations are publishing results?"
+                  body="Open the developer pages to see which organizations appear most often in the reported records."
                   href="/developers"
                 />
               </div>
 
               <div className="mt-5 grid gap-3 border-t border-border/60 pt-5 sm:grid-cols-2">
                 <QuietStat label="Models" value={models.length.toString()} detail="Tracked in the current corpus" tone="amber" />
-                <QuietStat label="Evaluations" value={evalSummaries.length.toString()} detail="Benchmark views with linked details" tone="sky" />
+                <QuietStat label="Evaluations" value={evalSummaries.length.toString()} detail="Benchmark records with linked details" tone="sky" />
                 <QuietStat label="Developers" value={developerCount.toString()} detail="Organizations represented" tone="emerald" />
-                <QuietStat label="Reported results" value={totalReportedResults.toLocaleString()} detail={`Avg ${avgBenchmarksPerModel.toFixed(1)} benchmark suites per model`} tone="slate" />
+                <QuietStat label="Reported results" value={totalReportedResults.toLocaleString()} detail={`Avg ${avgBenchmarksPerModel.toFixed(1)} reported benchmarks per model`} tone="slate" />
               </div>
             </aside>
           </div>
@@ -151,20 +151,20 @@ export default async function HomePage() {
               <RoutePanel
                 href="/models"
                 icon={<Database className="h-4 w-4" />}
-                title="Model-first reading"
-                body="See the reported benchmark footprint of a model, including where evidence is broad, narrow, or missing."
+                title="Model records"
+                body="See which benchmarks are reported for a model and where reporting is missing or thin."
               />
               <RoutePanel
                 href="/evals"
                 icon={<BookOpenText className="h-4 w-4" />}
-                title="Benchmark-first reading"
-                body="Inspect how a benchmark is reported across models, with room to compare slices, setups, and sources."
+                title="Benchmark records"
+                body="Inspect how one benchmark is reported across models, including slices, setup notes, and sources."
               />
               <RoutePanel
                 href="/about"
                 icon={<Scale className="h-4 w-4" />}
-                title="Project framing"
-                body="Read why this reporting format exists and how it supports both research and policy reading modes."
+                title="Project notes"
+                body="Read why this reporting format exists and how the same records support research and policy use."
               />
             </div>
 
@@ -200,7 +200,7 @@ function InquiryRow({
   href: string
 }) {
   return (
-    <Link href={href} className="group rounded-[1.3rem] border border-border/70 bg-muted/10 p-4 transition-colors hover:bg-muted/20">
+    <Link href={href} className="group block rounded-[1.3rem] border border-border/70 bg-muted/10 p-4 transition-colors hover:bg-muted/20">
       <div className="flex items-start justify-between gap-3">
         <div>
           <div className="text-sm font-semibold text-foreground">{title}</div>

components/eval-detail.tsx

@@ -300,6 +300,7 @@ export function EvalDetail({ summary }: EvalDetailProps) {
   const hasMultiMetricLeaderboard =
     (summary.leaderboard_metrics?.length ?? 0) > 1 &&
     (summary.leaderboard_rows?.length ?? 0) > 0
+  const [overviewOpen, setOverviewOpen] = useState(true)
   const [expandedRows, setExpandedRows] = useState<Record<string, boolean>>({})
   const [leaderboardPage, setLeaderboardPage] = useState(1)
   const [minParamStep, setMinParamStep] = useState(0)
@@ -409,203 +410,198 @@ export function EvalDetail({ summary }: EvalDetailProps) {
   return (
     <div className="space-y-6">
       <Card className="overflow-hidden">
-        <CardContent className="space-y-5 p-5 sm:p-6">
-          <div className="flex flex-col gap-5 xl:flex-row xl:items-start xl:justify-between">
-            <div className="space-y-3">
-              <div className="flex flex-wrap items-center gap-2">
-                <Badge variant="outline" className="border-border/60 bg-background/80 text-[11px] uppercase tracking-[0.18em]">
-                  {summary.is_aggregated ? "Merged Benchmark" : "Single Benchmark"}
-                </Badge>
-                {summary.is_aggregated ? (
+        <Collapsible open={overviewOpen} onOpenChange={setOverviewOpen}>
+          <CollapsibleTrigger asChild>
+            <button
+              type="button"
+              className="flex w-full items-center justify-between gap-4 border-b bg-muted/10 px-4 py-3 text-left transition-colors hover:bg-muted/15 sm:px-5"
+            >
+              <div className="min-w-0 space-y-1">
+                <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-muted-foreground">
+                  {isResearchView ? "Benchmark overview" : "Reading overview"}
+                </div>
+                <div className="flex flex-wrap items-center gap-2">
+                  <span className="text-base font-semibold tracking-tight sm:text-lg">{summary.evaluation_name}</span>
                   <Badge variant="secondary" className="font-normal">
-                    {summary.aggregate_sources?.length ?? 0} composite benchmarks
+                    {summary.models_count} models
                   </Badge>
-                ) : (
                   <Badge variant="secondary" className="font-normal">
-                    Composite: {summary.composite_benchmark_name}
+                    {hasMultiMetricLeaderboard
+                      ? `${summary.metrics_count ?? summary.leaderboard_metrics?.length ?? 1} measures`
+                      : `${summary.metrics_count ?? 1} ${(summary.metrics_count ?? 1) === 1 ? "measure" : "measures"}`}
                   </Badge>
-                )}
-                <Badge variant="secondary" className="font-normal capitalize">
-                  {summary.metric_config.score_type}
-                </Badge>
-                <Badge variant="secondary" className="font-normal">
-                  {summary.metric_config.lower_is_better ? "Lower is better" : "Higher is better"}
-                </Badge>
-                {summary.tags?.languages && summary.tags.languages.length > 0 && (
-                  <Badge variant="secondary" className="font-normal">
-                    {summary.tags.languages.join(", ")}
-                  </Badge>
-                )}
-              </div>
-
-              <div className="space-y-1">
-                <div className="text-2xl font-semibold tracking-tight sm:text-[1.9rem]">{summary.evaluation_name}</div>
-                <p className="max-w-3xl text-sm leading-6 text-muted-foreground">
-                  {summary.metric_config.evaluation_description}
-                </p>
+                </div>
               </div>
-
-              {!isResearchView && (
-                <p className="max-w-3xl text-sm leading-6 text-muted-foreground">
-                  {`${summary.benchmark_card?.purpose_and_intended_users?.goal ?? "This benchmark provides a public-facing capability signal."} Scores should be read alongside benchmark scope, metric definitions, and the source dataset context.`}
-                </p>
+              {overviewOpen ? (
+                <ChevronUp className="h-4 w-4 shrink-0 text-muted-foreground" />
+              ) : (
+                <ChevronDown className="h-4 w-4 shrink-0 text-muted-foreground" />
               )}
-            </div>
+            </button>
+          </CollapsibleTrigger>
+
+          <CollapsibleContent>
+            <CardContent className="space-y-4 p-4 sm:p-5">
+              <div className="flex flex-col gap-4 xl:flex-row xl:items-start xl:justify-between">
+                <div className="space-y-2.5">
+                  <div className="flex flex-wrap items-center gap-2">
+                    <Badge variant="outline" className="border-border/60 bg-background/80 text-[11px] uppercase tracking-[0.18em]">
+                      {summary.is_aggregated ? "Merged Benchmark" : "Single Benchmark"}
+                    </Badge>
+                    {summary.is_aggregated ? (
+                      <Badge variant="secondary" className="font-normal">
+                        {summary.aggregate_sources?.length ?? 0} composite benchmarks
+                      </Badge>
+                    ) : (
+                      <Badge variant="secondary" className="font-normal">
+                        Composite: {summary.composite_benchmark_name}
+                      </Badge>
+                    )}
+                    <Badge variant="secondary" className="font-normal capitalize">
+                      {summary.metric_config.score_type}
+                    </Badge>
+                    <Badge variant="secondary" className="font-normal">
+                      {summary.metric_config.lower_is_better ? "Lower is better" : "Higher is better"}
+                    </Badge>
+                    {summary.tags?.languages && summary.tags.languages.length > 0 && (
+                      <Badge variant="secondary" className="font-normal">
+                        {summary.tags.languages.join(", ")}
+                      </Badge>
+                    )}
+                  </div>
 
-            <div className="grid w-full gap-3 grid-cols-2 xl:grid-cols-4">
-              <div className="rounded-2xl border border-sky-200/80 bg-sky-50/80 px-4 py-3 dark:border-sky-900/40 dark:bg-sky-950/20">
-                <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-sky-700 dark:text-sky-200">Models</div>
-                <div className="mt-1 text-2xl font-semibold text-sky-950 dark:text-sky-50">{summary.models_count}</div>
-              </div>
-              <div className="rounded-2xl border border-border/70 bg-muted/20 px-4 py-3">
-                <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-muted-foreground">
-                  {hasMultiMetricLeaderboard ? "Metrics" : isResearchView ? "Avg score" : "Metrics"}
-                </div>
-                <div className="mt-1 text-2xl font-semibold">
-                  {hasMultiMetricLeaderboard ? summary.metrics_count ?? summary.leaderboard_metrics?.length ?? 1 : isResearchView ? avgScoreLabel : summary.metrics_count ?? 1}
-                </div>
-              </div>
-              <div className="rounded-2xl border border-emerald-200/80 bg-emerald-50/80 px-4 py-3 dark:border-emerald-900/40 dark:bg-emerald-950/20">
-                <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-emerald-700 dark:text-emerald-200">
-                  {hasMultiMetricLeaderboard || !isResearchView ? "Source dataset" : "Top model"}
-                </div>
-                <div className="mt-1 text-sm font-semibold text-emerald-950 dark:text-emerald-50">
-                  {hasMultiMetricLeaderboard || !isResearchView
-                    ? sourceDatasetLabel
-                    : summary.best_model?.name ?? "Unknown"}
+                  <p className="max-w-3xl text-sm leading-6 text-muted-foreground">
+                    {summary.metric_config.evaluation_description}
+                  </p>
+
+                  {!isResearchView && (
+                    <p className="max-w-3xl text-sm leading-6 text-muted-foreground">
+                      {`${summary.benchmark_card?.purpose_and_intended_users?.goal ?? "This benchmark reports a capability result."} Scores should be read alongside benchmark scope, metric definitions, and the source dataset context.`}
+                    </p>
+                  )}
                 </div>
-                {!hasMultiMetricLeaderboard && isResearchView && summary.best_model && (
-                  <div className="mt-1 text-xs text-emerald-700/80 dark:text-emerald-200/80">
-                    {formatRawScore(summary.best_model.score, summary.metric_config.unit)}
+
+                <div className="grid w-full grid-cols-2 gap-2 xl:grid-cols-4">
+                  <div className="rounded-xl border border-sky-200/80 bg-sky-50/80 px-3 py-2.5 dark:border-sky-900/40 dark:bg-sky-950/20">
+                    <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-sky-700 dark:text-sky-200">Models</div>
+                    <div className="mt-1 text-xl font-semibold text-sky-950 dark:text-sky-50">{summary.models_count}</div>
                   </div>
-                )}
-              </div>
-              <div className="rounded-2xl border border-amber-200/80 bg-amber-50/80 px-4 py-3 dark:border-amber-900/40 dark:bg-amber-950/20">
-                <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-amber-700 dark:text-amber-200">
-                  {hasMultiMetricLeaderboard || !isResearchView ? "Instance data" : "Bottom model"}
-                </div>
-                <div className="mt-1 text-sm font-semibold text-amber-950 dark:text-amber-50">
-                  {hasMultiMetricLeaderboard || !isResearchView
-                    ? instanceDataLabel
-                    : summary.worst_model?.name ?? "Unknown"}
-                </div>
-                {!hasMultiMetricLeaderboard && isResearchView && summary.worst_model && (
-                  <div className="mt-1 text-xs text-amber-700/80 dark:text-amber-200/80">
-                    {formatRawScore(summary.worst_model.score, summary.metric_config.unit)}
+                  <div className="rounded-xl border border-border/70 bg-muted/20 px-3 py-2.5">
+                    <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-muted-foreground">
+                      {hasMultiMetricLeaderboard ? "Measures" : isResearchView ? "Avg score" : "Measures"}
+                    </div>
+                    <div className="mt-1 text-xl font-semibold">
+                      {hasMultiMetricLeaderboard ? summary.metrics_count ?? summary.leaderboard_metrics?.length ?? 1 : isResearchView ? avgScoreLabel : summary.metrics_count ?? 1}
+                    </div>
                   </div>
-                )}
+                  <div className="rounded-xl border border-emerald-200/80 bg-emerald-50/80 px-3 py-2.5 dark:border-emerald-900/40 dark:bg-emerald-950/20">
+                    <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-emerald-700 dark:text-emerald-200">
+                      {hasMultiMetricLeaderboard || !isResearchView ? "Source dataset" : "Top model"}
+                    </div>
+                    <div className="mt-1 text-sm font-semibold text-emerald-950 dark:text-emerald-50">
+                      {hasMultiMetricLeaderboard || !isResearchView
+                        ? sourceDatasetLabel
+                        : summary.best_model?.name ?? "Unknown"}
+                    </div>
+                    {!hasMultiMetricLeaderboard && isResearchView && summary.best_model && (
+                      <div className="mt-1 text-xs text-emerald-700/80 dark:text-emerald-200/80">
+                        {formatRawScore(summary.best_model.score, summary.metric_config.unit)}
+                      </div>
+                    )}
+                  </div>
+                  <div className="rounded-xl border border-amber-200/80 bg-amber-50/80 px-3 py-2.5 dark:border-amber-900/40 dark:bg-amber-950/20">
+                    <div className="text-[11px] font-semibold uppercase tracking-[0.18em] text-amber-700 dark:text-amber-200">
+                      {hasMultiMetricLeaderboard || !isResearchView ? "Instance data" : "Bottom model"}
+                    </div>
+                    <div className="mt-1 text-sm font-semibold text-amber-950 dark:text-amber-50">
+                      {hasMultiMetricLeaderboard || !isResearchView
+                        ? instanceDataLabel
+                        : summary.worst_model?.name ?? "Unknown"}
+                    </div>
+                    {!hasMultiMetricLeaderboard && isResearchView && summary.worst_model && (
+                      <div className="mt-1 text-xs text-amber-700/80 dark:text-amber-200/80">
+                        {formatRawScore(summary.worst_model.score, summary.metric_config.unit)}
+                      </div>
+                    )}
+                  </div>
+                </div>
               </div>
-            </div>
-          </div>
 
-          <div className="rounded-[1.5rem] border bg-muted/10 p-4">
-            <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-muted-foreground">
-              {isResearchView ? "Metric specification" : "Reading context"}
-            </div>
-            <dl className="mt-3 grid gap-x-6 gap-y-3 text-sm sm:grid-cols-2 xl:grid-cols-5">
-              <div>
-                <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
-                  Composite benchmark
-                </dt>
-                <dd className="mt-1 break-words font-medium">
-                  {summary.is_aggregated
-                    ? summary.aggregate_sources?.map((source) => source.composite_benchmark_name).join(", ") || "Multiple composite benchmarks"
-                    : summary.composite_benchmark_name}
-                </dd>
-              </div>
-              <div>
-                <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
-                  {isResearchView ? "Single benchmark ID" : "What this covers"}
-                </dt>
-                <dd className="mt-1 break-words font-medium">
-                  {isResearchView
-                    ? summary.evaluation_id
-                    : summary.is_aggregated
-                      ? summary.metric_config.evaluation_description
-                      : summary.metric_config.evaluation_description}
-                </dd>
-              </div>
-              <div>
-                <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
-                  {isResearchView ? "Score scale" : "How to read scores"}
-                </dt>
-                <dd className="mt-1 font-medium">
-                  {isResearchView
-                    ? `${summary.metric_config.min_score ?? 0} - ${summary.metric_config.max_score ?? 1}`
-                    : scoreDirectionLabel}
-                </dd>
-              </div>
-              {summary.tags?.domains && summary.tags.domains.length > 0 && (
-                <div>
-                  <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">Domain coverage</dt>
-                  <dd className="mt-1 font-medium capitalize">
-                    {summary.tags.domains.slice(0, 2).join(", ")}
-                    {summary.tags.domains.length > 2 ? ` +${summary.tags.domains.length - 2} more` : ""}
-                  </dd>
+              <div className="rounded-2xl border bg-muted/10 p-3.5">
+                <div className="text-[11px] font-semibold uppercase tracking-[0.2em] text-muted-foreground">
+                  {isResearchView ? "Metric specification" : "Reading context"}
                 </div>
-              )}
-              <div>
-                <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
-                  {isResearchView ? "Source dataset" : "Instance data"}
-                </dt>
-                <dd className="mt-1 font-medium">
-                  {isResearchView ? sourceDatasetLabel : instanceDataLabel}
-                </dd>
+                <dl className="mt-3 grid gap-x-5 gap-y-3 text-sm sm:grid-cols-2 xl:grid-cols-5">
+                  <div>
+                    <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
+                      Composite benchmark
+                    </dt>
+                    <dd className="mt-1 break-words font-medium">
+                      {summary.is_aggregated
+                        ? summary.aggregate_sources?.map((source) => source.composite_benchmark_name).join(", ") || "Multiple composite benchmarks"
+                        : summary.composite_benchmark_name}
+                    </dd>
+                  </div>
+                  <div>
+                    <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
+                      {isResearchView ? "Single benchmark ID" : "What this covers"}
+                    </dt>
+                    <dd className="mt-1 break-words font-medium">
+                      {isResearchView ? summary.evaluation_id : summary.metric_config.evaluation_description}
+                    </dd>
+                  </div>
+                  <div>
+                    <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
+                      {isResearchView ? "Score scale" : "How to read scores"}
+                    </dt>
+                    <dd className="mt-1 font-medium">
+                      {isResearchView
+                        ? `${summary.metric_config.min_score ?? 0} - ${summary.metric_config.max_score ?? 1}`
+                        : scoreDirectionLabel}
+                    </dd>
+                  </div>
+                  {summary.tags?.domains && summary.tags.domains.length > 0 && (
+                    <div>
+                      <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">Domain tags</dt>
+                      <dd className="mt-1 font-medium capitalize">
+                        {summary.tags.domains.slice(0, 2).join(", ")}
+                        {summary.tags.domains.length > 2 ? ` +${summary.tags.domains.length - 2} more` : ""}
+                      </dd>
+                    </div>
+                  )}
+                  <div>
+                    <dt className="text-[11px] font-semibold uppercase tracking-[0.16em] text-muted-foreground">
+                      {isResearchView ? "Source dataset" : "Instance data"}
+                    </dt>
+                    <dd className="mt-1 font-medium">
+                      {isResearchView ? sourceDatasetLabel : instanceDataLabel}
+                    </dd>
+                  </div>
+                </dl>
               </div>
-            </dl>
-          </div>
-        </CardContent>
-      </Card>
 
-      {!hasMultiMetricLeaderboard && (summary.root_metrics?.length || summary.subtasks?.length) ? (
-        <Card className="overflow-hidden">
-          <CardHeader className="border-b bg-muted/10">
-            <CardTitle className="text-xl">Benchmark structure</CardTitle>
-            <CardDescription>
-              Benchmark-level summary metrics and benchmark breakdowns are shown as separate sections.
-            </CardDescription>
-          </CardHeader>
-          <CardContent className="space-y-6 p-5 sm:p-6">
-            {summary.root_metrics && summary.root_metrics.length > 0 && (
-              <section className="space-y-3">
-                <div>
-                  <div className="text-sm font-semibold">Benchmark-level metrics</div>
-                  <div className="text-xs text-muted-foreground">
-                    Benchmark summary metrics used in this evaluation view.
+              {!hasMultiMetricLeaderboard && (summary.root_metrics?.length || summary.subtasks?.length) ? (
+                <section className="rounded-2xl border bg-muted/5 p-3.5">
+                  <div className="space-y-1">
+                    <div className="text-sm font-semibold">Benchmark structure</div>
+                    <div className="text-xs text-muted-foreground">
+                      Benchmark-level summary metrics and subtask slices grouped in one compact section.
+                    </div>
                   </div>
-                </div>
-                <div className="flex flex-wrap gap-2">
-                  {summary.root_metrics.map((metric) => (
-                    <span
-                      key={metric.metric_summary_id}
-                      className="rounded-full border border-border/70 bg-background px-3 py-1.5 text-xs font-medium"
-                      title={metric.canonical_display_name || metric.display_name}
-                    >
-                      {getCompactMetricLabel(metric.display_name)}
-                      {typeof metric.top_score === "number" ? ` · ${formatRawScore(metric.top_score, metric.unit)}` : ""}
-                    </span>
-                  ))}
-                </div>
-              </section>
-            )}
 
-            {summary.subtasks && summary.subtasks.length > 0 && (
-              <section className="space-y-3">
-                <div>
-                  <div className="text-sm font-semibold">Subtask breakdown</div>
-                </div>
-                <div className="grid gap-3 lg:grid-cols-2">
-                  {summary.subtasks.map((subtask) => (
-                    <div key={subtask.subtask_key} className="rounded-2xl border bg-background p-4">
-                      <div className="font-semibold">{subtask.display_name || subtask.subtask_name}</div>
-                      <div className="mt-1 text-xs text-muted-foreground" title={subtask.canonical_display_name || subtask.display_name}>
-                        {subtask.canonical_display_name || subtask.display_name}
+                  {summary.root_metrics && summary.root_metrics.length > 0 && (
+                    <div className="mt-4 space-y-2.5">
+                      <div>
+                        <div className="text-sm font-semibold">Benchmark-level metrics</div>
+                        <div className="text-xs text-muted-foreground">
+                          Benchmark summary metrics used in this evaluation view.
+                        </div>
                       </div>
-                      <div className="mt-3 flex flex-wrap gap-2">
-                        {subtask.metrics.map((metric) => (
+                      <div className="flex flex-wrap gap-2">
+                        {summary.root_metrics.map((metric) => (
                           <span
                             key={metric.metric_summary_id}
-                            className="rounded-full border border-border/70 bg-muted/20 px-2.5 py-1 text-[11px] font-medium"
+                            className="rounded-full border border-border/70 bg-background px-3 py-1.5 text-xs font-medium"
                             title={metric.canonical_display_name || metric.display_name}
                           >
                             {getCompactMetricLabel(metric.display_name)}
@@ -614,18 +610,50 @@ export function EvalDetail({ summary }: EvalDetailProps) {
                         ))}
                       </div>
                     </div>
-                  ))}
-                </div>
-              </section>
-            )}
-          </CardContent>
-        </Card>
-      ) : null}
+                  )}
 
-      {/* Policy: benchmark context BEFORE the leaderboard (context first, numbers second) */}
-      {!isResearchView && summary.benchmark_card && (
-        <BenchmarkCardPanel card={summary.benchmark_card} isResearchView={false} defaultRisksOpen />
-      )}
+                  {summary.subtasks && summary.subtasks.length > 0 && (
+                    <div className="mt-4 space-y-2.5">
+                      <div className="text-sm font-semibold">Subtask breakdown</div>
+                      <div className="grid gap-3 lg:grid-cols-2">
+                        {summary.subtasks.map((subtask) => (
+                          <div key={subtask.subtask_key} className="rounded-xl border bg-background p-3.5">
+                            <div className="font-semibold">{subtask.display_name || subtask.subtask_name}</div>
+                            <div className="mt-1 text-xs text-muted-foreground" title={subtask.canonical_display_name || subtask.display_name}>
+                              {subtask.canonical_display_name || subtask.display_name}
+                            </div>
+                            <div className="mt-3 flex flex-wrap gap-2">
+                              {subtask.metrics.map((metric) => (
+                                <span
+                                  key={metric.metric_summary_id}
+                                  className="rounded-full border border-border/70 bg-muted/20 px-2.5 py-1 text-[11px] font-medium"
+                                  title={metric.canonical_display_name || metric.display_name}
+                                >
+                                  {getCompactMetricLabel(metric.display_name)}
+                                  {typeof metric.top_score === "number" ? ` · ${formatRawScore(metric.top_score, metric.unit)}` : ""}
+                                </span>
+                              ))}
+                            </div>
+                          </div>
+                        ))}
+                      </div>
+                    </div>
+                  )}
+                </section>
+              ) : null}
+
+              {summary.benchmark_card && (
+                <BenchmarkCardCollapsible
+                  card={summary.benchmark_card}
+                  isResearchView={isResearchView}
+                  defaultOpen
+                  defaultRisksOpen={!isResearchView}
+                />
+              )}
+            </CardContent>
+          </CollapsibleContent>
+        </Collapsible>
+      </Card>
 
       {hasMultiMetricLeaderboard ? (
         <MultiMetricLeaderboard summary={summary} isResearchView={isResearchView} />
@@ -1135,11 +1163,6 @@ export function EvalDetail({ summary }: EvalDetailProps) {
           </CardContent>
         </Card>
       )}
-
-      {/* Research: benchmark card details AFTER the leaderboard, collapsed by default */}
-      {isResearchView && summary.benchmark_card && (
-        <ResearchBenchmarkCardCollapsible card={summary.benchmark_card} />
-      )}
     </div>
   )
 }
@@ -1433,8 +1456,8 @@ function MultiMetricLeaderboard({
             </div>
             <CardDescription>
               {isResearchView
-                ? "Each column is a reported benchmark metric. Distinct measures stay separate instead of collapsing into a single raw score."
-                : "Each column is a separately reported metric or subtask metric so the benchmark can be read without flattening unlike measures into one number."}
+                ? "Each column is a reported benchmark measure. Distinct measures stay separate instead of collapsing into a single raw score."
+                : "Each column is a separately reported measure so the benchmark can be read without flattening different results into one number."}
             </CardDescription>
           </div>
 
@@ -1446,8 +1469,8 @@ function MultiMetricLeaderboard({
             </Badge>
             <Badge variant="outline">
               {visibleMetrics.length === leaderboardMetrics.length
-                ? `${leaderboardMetrics.length} metrics`
-                : `${visibleMetrics.length} of ${leaderboardMetrics.length} metrics`}
+                ? `${leaderboardMetrics.length} measures`
+                : `${visibleMetrics.length} of ${leaderboardMetrics.length} measures`}
             </Badge>
             {hasParameterData && (numericMinParams != null || numericMaxParams != null) && (
               <Badge variant="outline">
@@ -1462,7 +1485,7 @@ function MultiMetricLeaderboard({
                 </Button>
               </DropdownMenuTrigger>
               <DropdownMenuContent align="end" className="w-80">
-                <DropdownMenuLabel>Visible metric columns</DropdownMenuLabel>
+                <DropdownMenuLabel>Visible measure columns</DropdownMenuLabel>
                 <DropdownMenuItem onSelect={() => setVisibleMetricKeys(allMetricKeys)}>
                   Show all
                 </DropdownMenuItem>
@@ -1638,7 +1661,7 @@ function MultiMetricLeaderboard({
                     onClick={() => handleSort("coverage")}
                     className="w-full text-right font-semibold transition-colors hover:text-primary"
                   >
-                    Coverage{getSortIndicator("coverage")}
+                    Measures present{getSortIndicator("coverage")}
                   </button>
                 </TableHead>
                 {visibleMetrics.map((metric) => (
@@ -1773,8 +1796,18 @@ function MultiMetricLeaderboard({
   )
 }
 
-function ResearchBenchmarkCardCollapsible({ card }: { card: BenchmarkCard }) {
-  const [open, setOpen] = useState(false)
+function BenchmarkCardCollapsible({
+  card,
+  isResearchView,
+  defaultOpen = true,
+  defaultRisksOpen = false,
+}: {
+  card: BenchmarkCard
+  isResearchView: boolean
+  defaultOpen?: boolean
+  defaultRisksOpen?: boolean
+}) {
+  const [open, setOpen] = useState(defaultOpen)
   return (
     <Collapsible open={open} onOpenChange={setOpen}>
       <CollapsibleTrigger asChild>
@@ -1797,7 +1830,11 @@ function ResearchBenchmarkCardCollapsible({ card }: { card: BenchmarkCard }) {
         </button>
       </CollapsibleTrigger>
       <CollapsibleContent className="mt-2">
-        <BenchmarkCardPanel card={card} isResearchView defaultRisksOpen={false} />
+        <BenchmarkCardPanel
+          card={card}
+          isResearchView={isResearchView}
+          defaultRisksOpen={defaultRisksOpen}
+        />
       </CollapsibleContent>
     </Collapsible>
   )

components/page-loading-state.tsx

@@ -0,0 +1,120 @@
+"use client"
+
+import { useEffect, useMemo, useState } from "react"
+
+import { cn } from "@/lib/utils"
+
+export interface PageLoadingStage {
+  label: string
+  done: boolean
+}
+
+interface PageLoadingStateProps {
+  title: string
+  description?: string
+  stages: PageLoadingStage[]
+  className?: string
+}
+
+function clamp(value: number, min: number, max: number) {
+  return Math.min(max, Math.max(min, value))
+}
+
+function getTargetProgress(completedStages: number, totalStages: number) {
+  const start = 12
+  const ceiling = 94
+
+  if (totalStages <= 0) {
+    return start
+  }
+
+  const stageWidth = (ceiling - start) / totalStages
+  const hardTarget = start + completedStages * stageWidth
+
+  if (completedStages >= totalStages) {
+    return 100
+  }
+
+  return Math.min(ceiling, hardTarget + Math.min(stageWidth * 0.35, 12))
+}
+
+export function PageLoadingState({ title, description, stages, className }: PageLoadingStateProps) {
+  const safeStages = stages.length > 0 ? stages : [{ label: "Loading", done: false }]
+
+  const { completedStages, currentStageLabel, targetProgress } = useMemo(() => {
+    const completed = safeStages.filter((stage) => stage.done).length
+    const currentStage = safeStages.find((stage) => !stage.done)?.label ?? "Finalizing"
+
+    return {
+      completedStages: completed,
+      currentStageLabel: currentStage,
+      targetProgress: getTargetProgress(completed, safeStages.length),
+    }
+  }, [safeStages])
+
+  const [displayProgress, setDisplayProgress] = useState(() => clamp(targetProgress, 0, 100))
+
+  useEffect(() => {
+    setDisplayProgress((current) => {
+      if (targetProgress < current) {
+        return clamp(targetProgress, 0, 100)
+      }
+
+      return current
+    })
+  }, [targetProgress])
+
+  useEffect(() => {
+    if (Math.abs(displayProgress - targetProgress) < 0.5) {
+      if (displayProgress !== targetProgress) {
+        setDisplayProgress(targetProgress)
+      }
+      return
+    }
+
+    const interval = window.setInterval(() => {
+      setDisplayProgress((current) => {
+        const difference = targetProgress - current
+        if (Math.abs(difference) < 0.5) {
+          return targetProgress
+        }
+
+        const increment =
+          difference > 18 ? 6 : difference > 10 ? 4 : difference > 4 ? 2 : 1
+
+        return clamp(current + increment, 0, targetProgress)
+      })
+    }, 110)
+
+    return () => {
+      window.clearInterval(interval)
+    }
+  }, [displayProgress, targetProgress])
+
+  const progressStyle = {
+    background: `conic-gradient(from 180deg, hsl(var(--primary)) 0deg ${displayProgress * 3.6}deg, hsl(var(--border)) ${displayProgress * 3.6}deg 360deg)`,
+  }
+
+  return (
+    <div className={cn("flex min-h-[20rem] items-center justify-center px-4", className)}>
+      <section className="flex w-full max-w-md flex-col items-center gap-5 rounded-[1.75rem] border border-border/60 bg-background/95 px-6 py-8 text-center shadow-[0_24px_70px_-52px_rgba(15,23,42,0.5)] backdrop-blur">
+        <div className="relative flex h-28 w-28 items-center justify-center rounded-full" style={progressStyle}>
+          <div className="flex h-[5.4rem] w-[5.4rem] items-center justify-center rounded-full border border-border/60 bg-background text-2xl font-semibold tracking-tight text-foreground tabular-nums">
+            {Math.round(displayProgress)}%
+          </div>
+        </div>
+
+        <div className="space-y-1.5">
+          <h2 className="text-xl font-semibold tracking-tight text-foreground sm:text-2xl">{title}</h2>
+          {description ? (
+            <p className="text-sm leading-6 text-muted-foreground">{description}</p>
+          ) : null}
+        </div>
+
+        <p className="text-[11px] font-medium uppercase tracking-[0.22em] text-muted-foreground">
+          {currentStageLabel} · {completedStages}/{safeStages.length} ready
+        </p>
+      </section>
+    </div>
+  )
+}

data/benchmarks/ace.json

@@ -1,120 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/Opus 4.1",
-      "name": "Opus 4.1",
-      "developer": "Anthropic",
-      "scores": {
-        "Overall Score": 0.4,
-        "Gaming Score": 0.318
-      }
-    },
-    {
-      "model_id": "anthropic/Opus 4.5",
-      "name": "Opus 4.5",
-      "developer": "Anthropic",
-      "scores": {
-        "Overall Score": 0.478,
-        "Gaming Score": 0.391
-      }
-    },
-    {
-      "model_id": "anthropic/Sonnet 4.5",
-      "name": "Sonnet 4.5",
-      "developer": "Anthropic",
-      "scores": {
-        "Overall Score": 0.44,
-        "Gaming Score": 0.373
-      }
-    },
-    {
-      "model_id": "google/Gemini 2.5 Flash",
-      "name": "Gemini 2.5 Flash",
-      "developer": "Google",
-      "scores": {
-        "Overall Score": 0.38,
-        "Gaming Score": 0.284
-      }
-    },
-    {
-      "model_id": "google/Gemini 2.5 Pro",
-      "name": "Gemini 2.5 Pro",
-      "developer": "Google",
-      "scores": {
-        "Overall Score": 0.4,
-        "Gaming Score": 0.285
-      }
-    },
-    {
-      "model_id": "google/Gemini 3 Flash",
-      "name": "Gemini 3 Flash",
-      "developer": "Google",
-      "scores": {
-        "Gaming Score": 0.415
-      }
-    },
-    {
-      "model_id": "google/Gemini 3 Pro",
-      "name": "Gemini 3 Pro",
-      "developer": "Google",
-      "scores": {
-        "Overall Score": 0.47,
-        "Gaming Score": 0.509
-      }
-    },
-    {
-      "model_id": "openai/GPT 5",
-      "name": "GPT 5",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.561,
-        "DIY Score": 0.55,
-        "Food Score": 0.7,
-        "Gaming Score": 0.575
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.1",
-      "name": "GPT 5.1",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.551,
-        "DIY Score": 0.56,
-        "Gaming Score": 0.61,
-        "Shopping Score": 0.45
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.2",
-      "name": "GPT 5.2",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.515,
-        "Food Score": 0.65,
-        "Gaming Score": 0.578
-      }
-    },
-    {
-      "model_id": "openai/o3",
-      "name": "o3",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.529,
-        "Gaming Score": 0.585,
-        "Shopping Score": 0.45
-      }
-    },
-    {
-      "model_id": "openai/o3 Pro",
-      "name": "o3 Pro",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.552,
-        "DIY Score": 0.54,
-        "Food Score": 0.6,
-        "Gaming Score": 0.613,
-        "Shopping Score": 0.45
-      }
-    }
-  ]
-}

data/benchmarks/apex-agents.json

@@ -1,218 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/Opus 4.5",
-      "name": "Opus 4.5",
-      "developer": "Anthropic",
-      "scores": {
-        "Overall Pass@1": 0.184,
-        "Overall Pass@8": 0.34,
-        "Overall Mean Score": 0.348,
-        "Investment Banking Pass@1": 0.216,
-        "Management Consulting Pass@1": 0.132,
-        "Corporate Law Pass@1": 0.202,
-        "Corporate Lawyer Mean Score": 0.471
-      }
-    },
-    {
-      "model_id": "anthropic/Opus 4.6",
-      "name": "Opus 4.6",
-      "developer": "Anthropic",
-      "scores": {
-        "Overall Pass@1": 0.298,
-        "Corporate Lawyer Mean Score": 0.502
-      }
-    },
-    {
-      "model_id": "applied-compute/Applied Compute: Small",
-      "name": "Applied Compute: Small",
-      "developer": "applied-compute",
-      "scores": {
-        "Overall Pass@1": 0.23,
-        "Overall Mean Score": 0.401,
-        "Corporate Law Pass@1": 0.266,
-        "Corporate Lawyer Mean Score": 0.548
-      }
-    },
-    {
-      "model_id": "google/Gemini 3 Flash",
-      "name": "Gemini 3 Flash",
-      "developer": "Google",
-      "scores": {
-        "Overall Pass@1": 0.24,
-        "Overall Pass@8": 0.367,
-        "Overall Mean Score": 0.395,
-        "Investment Banking Pass@1": 0.267,
-        "Management Consulting Pass@1": 0.193,
-        "Corporate Law Pass@1": 0.259,
-        "Corporate Lawyer Mean Score": 0.524
-      }
-    },
-    {
-      "model_id": "google/Gemini 3 Pro",
-      "name": "Gemini 3 Pro",
-      "developer": "Google",
-      "scores": {
-        "Overall Pass@1": 0.184,
-        "Overall Pass@8": 0.373,
-        "Overall Mean Score": 0.341,
-        "Investment Banking Pass@1": 0.188,
-        "Management Consulting Pass@1": 0.124,
-        "Corporate Law Pass@1": 0.239,
-        "Corporate Lawyer Mean Score": 0.487
-      }
-    },
-    {
-      "model_id": "google/Gemini 3.1 Pro",
-      "name": "Gemini 3.1 Pro",
-      "developer": "Google",
-      "scores": {
-        "Overall Pass@1": 0.335,
-        "Corporate Lawyer Mean Score": 0.494
-      }
-    },
-    {
-      "model_id": "minimax/Minimax-2.5",
-      "name": "Minimax-2.5",
-      "developer": "minimax",
-      "scores": {
-        "Corporate Lawyer Mean Score": 0.339
-      }
-    },
-    {
-      "model_id": "moonshot/Kimi K2 Thinking",
-      "name": "Kimi K2 Thinking",
-      "developer": "moonshot",
-      "scores": {
-        "Overall Pass@1": 0.04,
-        "Overall Pass@8": 0.144,
-        "Overall Mean Score": 0.115,
-        "Investment Banking Pass@1": 0.012,
-        "Management Consulting Pass@1": 0.029,
-        "Corporate Law Pass@1": 0.08,
-        "Corporate Lawyer Mean Score": 0.223
-      }
-    },
-    {
-      "model_id": "moonshot/Kimi K2.5",
-      "name": "Kimi K2.5",
-      "developer": "moonshot",
-      "scores": {
-        "Corporate Lawyer Mean Score": 0.402
-      }
-    },
-    {
-      "model_id": "openai/GPT 5",
-      "name": "GPT 5",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Pass@1": 0.183,
-        "Overall Pass@8": 0.31,
-        "Overall Mean Score": 0.329,
-        "Investment Banking Pass@1": 0.273,
-        "Management Consulting Pass@1": 0.123,
-        "Corporate Law Pass@1": 0.153,
-        "Corporate Lawyer Mean Score": 0.382
-      }
-    },
-    {
-      "model_id": "openai/GPT 5 Codex",
-      "name": "GPT 5 Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "Corporate Lawyer Mean Score": 0.362
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.1",
-      "name": "GPT 5.1",
-      "developer": "OpenAI",
-      "scores": {
-        "Corporate Lawyer Mean Score": 0.376
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.1 Codex",
-      "name": "GPT 5.1 Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "Corporate Lawyer Mean Score": 0.366
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.2",
-      "name": "GPT 5.2",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Pass@1": 0.23,
-        "Overall Pass@8": 0.4,
-        "Overall Mean Score": 0.387,
-        "Investment Banking Pass@1": 0.273,
-        "Management Consulting Pass@1": 0.227,
-        "Corporate Law Pass@1": 0.189,
-        "Corporate Lawyer Mean Score": 0.443
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.2 Codex",
-      "name": "GPT 5.2 Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Pass@1": 0.276,
-        "Corporate Lawyer Mean Score": 0.394
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.3 Codex",
-      "name": "GPT 5.3 Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Pass@1": 0.317
-      }
-    },
-    {
-      "model_id": "openai/GPT OSS 120B",
-      "name": "GPT OSS 120B",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Pass@1": 0.047,
-        "Overall Pass@8": 0.115,
-        "Overall Mean Score": 0.145,
-        "Investment Banking Pass@1": 0.027,
-        "Management Consulting Pass@1": 0.035,
-        "Corporate Law Pass@1": 0.078,
-        "Corporate Lawyer Mean Score": 0.269
-      }
-    },
-    {
-      "model_id": "xai/Grok 4",
-      "name": "Grok 4",
-      "developer": "xAI",
-      "scores": {
-        "Overall Pass@1": 0.152,
-        "Overall Pass@8": 0.329,
-        "Overall Mean Score": 0.303,
-        "Investment Banking Pass@1": 0.17,
-        "Management Consulting Pass@1": 0.12,
-        "Corporate Law Pass@1": 0.165,
-        "Corporate Lawyer Mean Score": 0.41
-      }
-    },
-    {
-      "model_id": "zhipu/GLM 4.6",
-      "name": "GLM 4.6",
-      "developer": "zhipu",
-      "scores": {
-        "Corporate Lawyer Mean Score": 0.196
-      }
-    },
-    {
-      "model_id": "zhipu/GLM 4.7",
-      "name": "GLM 4.7",
-      "developer": "zhipu",
-      "scores": {
-        "Corporate Lawyer Mean Score": 0.147
-      }
-    }
-  ]
-}

data/benchmarks/apex-v1.json

@@ -1,93 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/Opus 4.5",
-      "name": "Opus 4.5",
-      "developer": "Anthropic",
-      "scores": {
-        "Medicine (MD) Score": 0.65
-      }
-    },
-    {
-      "model_id": "google/Gemini 2.5 Flash",
-      "name": "Gemini 2.5 Flash",
-      "developer": "Google",
-      "scores": {
-        "Overall Score": 0.604
-      }
-    },
-    {
-      "model_id": "google/Gemini 3 Flash",
-      "name": "Gemini 3 Flash",
-      "developer": "Google",
-      "scores": {
-        "Overall Score": 0.64,
-        "Consulting Score": 0.64
-      }
-    },
-    {
-      "model_id": "google/Gemini 3 Pro",
-      "name": "Gemini 3 Pro",
-      "developer": "Google",
-      "scores": {
-        "Overall Score": 0.643,
-        "Consulting Score": 0.64,
-        "Investment Banking Score": 0.63
-      }
-    },
-    {
-      "model_id": "openai/GPT 4o",
-      "name": "GPT 4o",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.359
-      }
-    },
-    {
-      "model_id": "openai/GPT 5",
-      "name": "GPT 5",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.67,
-        "Big Law Score": 0.78,
-        "Medicine (MD) Score": 0.66,
-        "Investment Banking Score": 0.61
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.1",
-      "name": "GPT 5.1",
-      "developer": "OpenAI",
-      "scores": {
-        "Big Law Score": 0.77
-      }
-    },
-    {
-      "model_id": "openai/GPT 5.2 Pro",
-      "name": "GPT 5.2 Pro",
-      "developer": "OpenAI",
-      "scores": {
-        "Overall Score": 0.668,
-        "Consulting Score": 0.64,
-        "Medicine (MD) Score": 0.65,
-        "Investment Banking Score": 0.64
-      }
-    },
-    {
-      "model_id": "openai/o3",
-      "name": "o3",
-      "developer": "OpenAI",
-      "scores": {
-        "Big Law Score": 0.76
-      }
-    },
-    {
-      "model_id": "xai/Grok 4",
-      "name": "Grok 4",
-      "developer": "xAI",
-      "scores": {
-        "Overall Score": 0.635
-      }
-    }
-  ]
-}

data/benchmarks/appworld_test_normal.json

@@ -1,28 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/claude-opus-4-5",
-      "name": "claude-opus-4-5",
-      "developer": "Anthropic",
-      "scores": {
-        "appworld/test_normal": 0.68
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro-preview",
-      "name": "gemini-3-pro-preview",
-      "developer": "Google",
-      "scores": {
-        "appworld/test_normal": 0.505
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-2025-12-11",
-      "name": "gpt-5.2-2025-12-11",
-      "developer": "OpenAI",
-      "scores": {
-        "appworld/test_normal": 0.0
-      }
-    }
-  ]
-}

data/benchmarks/browsecompplus.json

@@ -1,28 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/claude-opus-4-5",
-      "name": "claude-opus-4-5",
-      "developer": "Anthropic",
-      "scores": {
-        "browsecompplus": 0.61
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro-preview",
-      "name": "gemini-3-pro-preview",
-      "developer": "Google",
-      "scores": {
-        "browsecompplus": 0.48
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-2025-12-11",
-      "name": "gpt-5.2-2025-12-11",
-      "developer": "OpenAI",
-      "scores": {
-        "browsecompplus": 0.26
-      }
-    }
-  ]
-}

data/benchmarks/global-mmlu-lite.json

@@ -1,706 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "alibaba/qwen3-235b-a22b-instruct-2507",
-      "name": "qwen3-235b-a22b-instruct-2507",
-      "developer": "alibaba",
-      "scores": {
-        "Global MMLU Lite": 0.8798,
-        "Culturally Sensitive": 0.8522,
-        "Culturally Agnostic": 0.9075,
-        "Arabic": 0.88,
-        "English": 0.89,
-        "Bengali": 0.8875,
-        "German": 0.885,
-        "French": 0.88,
-        "Hindi": 0.8775,
-        "Indonesian": 0.88,
-        "Italian": 0.88,
-        "Japanese": 0.88,
-        "Korean": 0.875,
-        "Portuguese": 0.8875,
-        "Spanish": 0.875,
-        "Swahili": 0.87,
-        "Yoruba": 0.8725,
-        "Chinese": 0.8775,
-        "Burmese": 0.88
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-5-haiku-20241022",
-      "name": "Claude 3.5 Haiku 20241022",
-      "developer": "Anthropic",
-      "scores": {
-        "Global MMLU Lite": 0.6114,
-        "Culturally Sensitive": 0.5834,
-        "Culturally Agnostic": 0.6394,
-        "Arabic": 0.695,
-        "English": 0.485,
-        "Bengali": 0.675,
-        "German": 0.565,
-        "French": 0.61,
-        "Hindi": 0.6575,
-        "Indonesian": 0.5475,
-        "Italian": 0.48,
-        "Japanese": 0.655,
-        "Korean": 0.6575,
-        "Portuguese": 0.5225,
-        "Spanish": 0.485,
-        "Swahili": 0.69,
-        "Yoruba": 0.6675,
-        "Chinese": 0.69,
-        "Burmese": 0.7
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-7-sonnet-20250219",
-      "name": "claude-3-7-sonnet-20250219",
-      "developer": "Anthropic",
-      "scores": {
-        "Global MMLU Lite": 0.8078,
-        "Culturally Sensitive": 0.7794,
-        "Culturally Agnostic": 0.8362,
-        "Arabic": 0.7925,
-        "English": 0.7625,
-        "Bengali": 0.825,
-        "German": 0.8125,
-        "French": 0.7675,
-        "Hindi": 0.805,
-        "Indonesian": 0.8175,
-        "Italian": 0.8225,
-        "Japanese": 0.8425,
-        "Korean": 0.83,
-        "Portuguese": 0.77,
-        "Spanish": 0.8075,
-        "Swahili": 0.8125,
-        "Yoruba": 0.81,
-        "Chinese": 0.835,
-        "Burmese": 0.8125
-      }
-    },
-    {
-      "model_id": "anthropic/claude-opus-4-1-20250805",
-      "name": "claude-opus-4-1-20250805",
-      "developer": "Anthropic",
-      "scores": {
-        "Global MMLU Lite": 0.943,
-        "Culturally Sensitive": 0.9331,
-        "Culturally Agnostic": 0.9528,
-        "Arabic": 0.945,
-        "English": 0.9475,
-        "Bengali": 0.9425,
-        "German": 0.94,
-        "French": 0.945,
-        "Hindi": 0.9475,
-        "Indonesian": 0.9425,
-        "Italian": 0.94,
-        "Japanese": 0.94,
-        "Korean": 0.95,
-        "Portuguese": 0.945,
-        "Spanish": 0.945,
-        "Swahili": 0.93,
-        "Yoruba": 0.9375,
-        "Chinese": 0.945,
-        "Burmese": 0.945
-      }
-    },
-    {
-      "model_id": "anthropic/claude-sonnet-4-20250514",
-      "name": "claude-sonnet-4-20250514",
-      "developer": "Anthropic",
-      "scores": {
-        "Global MMLU Lite": 0.9058,
-        "Culturally Sensitive": 0.8913,
-        "Culturally Agnostic": 0.9203,
-        "Arabic": 0.9125,
-        "English": 0.905,
-        "Bengali": 0.9075,
-        "German": 0.9125,
-        "French": 0.91,
-        "Hindi": 0.9,
-        "Indonesian": 0.9025,
-        "Italian": 0.9075,
-        "Japanese": 0.9,
-        "Korean": 0.9125,
-        "Portuguese": 0.91,
-        "Spanish": 0.9075,
-        "Swahili": 0.8975,
-        "Yoruba": 0.8975,
-        "Chinese": 0.9175,
-        "Burmese": 0.8925
-      }
-    },
-    {
-      "model_id": "cohere/aya-expanse-32b",
-      "name": "aya-expanse-32b",
-      "developer": "cohere",
-      "scores": {
-        "Global MMLU Lite": 0.7353,
-        "Culturally Sensitive": 0.6891,
-        "Culturally Agnostic": 0.7815,
-        "Arabic": 0.7425,
-        "English": 0.7544,
-        "Bengali": 0.7343,
-        "German": 0.7425,
-        "French": 0.7325,
-        "Hindi": 0.7375,
-        "Indonesian": 0.7594,
-        "Italian": 0.7305,
-        "Japanese": 0.7419,
-        "Korean": 0.7525,
-        "Portuguese": 0.7544,
-        "Spanish": 0.7362,
-        "Swahili": 0.7071,
-        "Yoruba": 0.6942,
-        "Chinese": 0.743,
-        "Burmese": 0.7025
-      }
-    },
-    {
-      "model_id": "cohere/command-a-03-2025",
-      "name": "command-a-03-2025",
-      "developer": "cohere",
-      "scores": {
-        "Global MMLU Lite": 0.8385,
-        "Culturally Sensitive": 0.7993,
-        "Culturally Agnostic": 0.8778,
-        "Arabic": 0.8425,
-        "English": 0.855,
-        "Bengali": 0.8225,
-        "German": 0.8425,
-        "French": 0.8375,
-        "Hindi": 0.8421,
-        "Indonesian": 0.8546,
-        "Italian": 0.8375,
-        "Japanese": 0.845,
-        "Korean": 0.85,
-        "Portuguese": 0.84,
-        "Spanish": 0.8525,
-        "Swahili": 0.8275,
-        "Yoruba": 0.815,
-        "Chinese": 0.835,
-        "Burmese": 0.8175
-      }
-    },
-    {
-      "model_id": "deepseek/deepseek-r1-0528",
-      "name": "deepseek-r1-0528",
-      "developer": "deepseek",
-      "scores": {
-        "Global MMLU Lite": 0.6744,
-        "Culturally Sensitive": 0.6672,
-        "Culturally Agnostic": 0.6816,
-        "Arabic": 0.6825,
-        "English": 0.715,
-        "Bengali": 0.655,
-        "German": 0.6375,
-        "French": 0.6925,
-        "Hindi": 0.6475,
-        "Indonesian": 0.655,
-        "Italian": 0.6775,
-        "Japanese": 0.7725,
-        "Korean": 0.6575,
-        "Portuguese": 0.635,
-        "Spanish": 0.7175,
-        "Swahili": 0.6775,
-        "Yoruba": 0.77,
-        "Chinese": 0.5075,
-        "Burmese": 0.69
-      }
-    },
-    {
-      "model_id": "deepseek/deepseek-v3.1",
-      "name": "deepseek-v3.1",
-      "developer": "deepseek",
-      "scores": {
-        "Global MMLU Lite": 0.8044,
-        "Culturally Sensitive": 0.7793,
-        "Culturally Agnostic": 0.8295,
-        "Arabic": 0.805,
-        "English": 0.825,
-        "Bengali": 0.8157,
-        "German": 0.7925,
-        "French": 0.8175,
-        "Hindi": 0.7569,
-        "Indonesian": 0.7764,
-        "Italian": 0.8075,
-        "Japanese": 0.8312,
-        "Korean": 0.8125,
-        "Portuguese": 0.8246,
-        "Spanish": 0.8125,
-        "Swahili": 0.801,
-        "Yoruba": 0.7831,
-        "Chinese": 0.8161,
-        "Burmese": 0.7925
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-flash",
-      "name": "Gemini 2.5 Flash",
-      "developer": "Google",
-      "scores": {
-        "Global MMLU Lite": 0.9145,
-        "Culturally Sensitive": 0.9,
-        "Culturally Agnostic": 0.9291,
-        "Arabic": 0.9125,
-        "English": 0.9325,
-        "Bengali": 0.91,
-        "German": 0.9025,
-        "French": 0.91,
-        "Hindi": 0.925,
-        "Indonesian": 0.9075,
-        "Italian": 0.9225,
-        "Japanese": 0.9125,
-        "Korean": 0.915,
-        "Portuguese": 0.9125,
-        "Spanish": 0.9175,
-        "Swahili": 0.915,
-        "Yoruba": 0.9075,
-        "Chinese": 0.915,
-        "Burmese": 0.915
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-flash-preview-05-20",
-      "name": "gemini-2.5-flash-preview-05-20",
-      "developer": "Google",
-      "scores": {
-        "Global MMLU Lite": 0.9092,
-        "Culturally Sensitive": 0.8925,
-        "Culturally Agnostic": 0.9259,
-        "Arabic": 0.905,
-        "English": 0.9225,
-        "Bengali": 0.91,
-        "German": 0.905,
-        "French": 0.925,
-        "Hindi": 0.9125,
-        "Indonesian": 0.9075,
-        "Italian": 0.89,
-        "Japanese": 0.9125,
-        "Korean": 0.9075,
-        "Portuguese": 0.915,
-        "Spanish": 0.915,
-        "Swahili": 0.905,
-        "Yoruba": 0.8825,
-        "Chinese": 0.93,
-        "Burmese": 0.9025
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-pro",
-      "name": "Gemini 2.5 Pro",
-      "developer": "Google",
-      "scores": {
-        "Global MMLU Lite": 0.9323,
-        "Culturally Sensitive": 0.9241,
-        "Culturally Agnostic": 0.9406,
-        "Arabic": 0.9475,
-        "English": 0.9275,
-        "Bengali": 0.9275,
-        "German": 0.93,
-        "French": 0.9425,
-        "Hindi": 0.9275,
-        "Indonesian": 0.925,
-        "Italian": 0.935,
-        "Japanese": 0.9375,
-        "Korean": 0.9275,
-        "Portuguese": 0.93,
-        "Spanish": 0.94,
-        "Swahili": 0.9375,
-        "Yoruba": 0.925,
-        "Chinese": 0.9275,
-        "Burmese": 0.93
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro-preview",
-      "name": "gemini-3-pro-preview",
-      "developer": "Google",
-      "scores": {
-        "Global MMLU Lite": 0.9453,
-        "Culturally Sensitive": 0.9397,
-        "Culturally Agnostic": 0.9509,
-        "Arabic": 0.9475,
-        "English": 0.9425,
-        "Bengali": 0.9425,
-        "German": 0.94,
-        "French": 0.9575,
-        "Hindi": 0.9425,
-        "Indonesian": 0.955,
-        "Italian": 0.955,
-        "Japanese": 0.94,
-        "Korean": 0.94,
-        "Portuguese": 0.9425,
-        "Spanish": 0.9475,
-        "Swahili": 0.94,
-        "Yoruba": 0.9425,
-        "Chinese": 0.9475,
-        "Burmese": 0.9425
-      }
-    },
-    {
-      "model_id": "google/gemma-3-27b-it",
-      "name": "gemma-3-27b-it",
-      "developer": "Google",
-      "scores": {
-        "Global MMLU Lite": 0.763,
-        "Culturally Sensitive": 0.7528,
-        "Culturally Agnostic": 0.7733,
-        "Arabic": 0.78,
-        "English": 0.7337,
-        "Bengali": 0.75,
-        "German": 0.775,
-        "French": 0.7481,
-        "Hindi": 0.7335,
-        "Indonesian": 0.7563,
-        "Italian": 0.75,
-        "Japanese": 0.7925,
-        "Korean": 0.798,
-        "Portuguese": 0.7481,
-        "Spanish": 0.7494,
-        "Swahili": 0.785,
-        "Yoruba": 0.7444,
-        "Chinese": 0.7925,
-        "Burmese": 0.7719
-      }
-    },
-    {
-      "model_id": "google/gemma-3-4b-it",
-      "name": "gemma-3-4b-it",
-      "developer": "Google",
-      "scores": {
-        "Global MMLU Lite": 0.6511,
-        "Culturally Sensitive": 0.6116,
-        "Culturally Agnostic": 0.6906,
-        "Arabic": 0.6525,
-        "English": 0.67,
-        "Bengali": 0.68,
-        "German": 0.6525,
-        "French": 0.6575,
-        "Hindi": 0.6475,
-        "Indonesian": 0.6775,
-        "Italian": 0.6675,
-        "Japanese": 0.6325,
-        "Korean": 0.66,
-        "Portuguese": 0.68,
-        "Spanish": 0.6725,
-        "Swahili": 0.6075,
-        "Yoruba": 0.5825,
-        "Chinese": 0.6475,
-        "Burmese": 0.63
-      }
-    },
-    {
-      "model_id": "ibm/granite-4.0-h-small",
-      "name": "granite-4.0-h-small",
-      "developer": "ibm",
-      "scores": {
-        "Global MMLU Lite": 0.7503,
-        "Culturally Sensitive": 0.7182,
-        "Culturally Agnostic": 0.7826,
-        "Arabic": 0.7613,
-        "English": 0.77,
-        "Bengali": 0.7613,
-        "German": 0.755,
-        "French": 0.7594,
-        "Hindi": 0.7575,
-        "Indonesian": 0.7614,
-        "Italian": 0.7525,
-        "Japanese": 0.7406,
-        "Korean": 0.7525,
-        "Portuguese": 0.757,
-        "Spanish": 0.7638,
-        "Swahili": 0.7318,
-        "Yoruba": 0.6921,
-        "Chinese": 0.7475,
-        "Burmese": 0.7419
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-medium-3",
-      "name": "mistral-medium-3",
-      "developer": "mistralai",
-      "scores": {
-        "Global MMLU Lite": 0.5511,
-        "Culturally Sensitive": 0.5391,
-        "Culturally Agnostic": 0.5631,
-        "Arabic": 0.455,
-        "English": 0.38,
-        "Bengali": 0.5175,
-        "German": 0.4775,
-        "French": 0.41,
-        "Hindi": 0.555,
-        "Indonesian": 0.515,
-        "Italian": 0.535,
-        "Japanese": 0.58,
-        "Korean": 0.595,
-        "Portuguese": 0.5175,
-        "Spanish": 0.5375,
-        "Swahili": 0.7075,
-        "Yoruba": 0.7675,
-        "Chinese": 0.535,
-        "Burmese": 0.7325
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-small-2503",
-      "name": "mistral-small-2503",
-      "developer": "mistralai",
-      "scores": {
-        "Global MMLU Lite": 0.7852,
-        "Culturally Sensitive": 0.7537,
-        "Culturally Agnostic": 0.8166,
-        "Arabic": 0.7875,
-        "English": 0.8,
-        "Bengali": 0.7725,
-        "German": 0.7975,
-        "French": 0.8,
-        "Hindi": 0.795,
-        "Indonesian": 0.785,
-        "Italian": 0.805,
-        "Japanese": 0.77,
-        "Korean": 0.79,
-        "Portuguese": 0.7925,
-        "Spanish": 0.7825,
-        "Swahili": 0.775,
-        "Yoruba": 0.735,
-        "Chinese": 0.7925,
-        "Burmese": 0.7825
-      }
-    },
-    {
-      "model_id": "openai/gpt-4.1-2025-04-14",
-      "name": "gpt-4.1-2025-04-14",
-      "developer": "OpenAI",
-      "scores": {
-        "Global MMLU Lite": 0.8755,
-        "Culturally Sensitive": 0.8541,
-        "Culturally Agnostic": 0.8969,
-        "Arabic": 0.88,
-        "English": 0.8825,
-        "Bengali": 0.8625,
-        "German": 0.875,
-        "French": 0.8875,
-        "Hindi": 0.8775,
-        "Indonesian": 0.885,
-        "Italian": 0.88,
-        "Japanese": 0.8725,
-        "Korean": 0.87,
-        "Portuguese": 0.875,
-        "Spanish": 0.885,
-        "Swahili": 0.8725,
-        "Yoruba": 0.875,
-        "Chinese": 0.87,
-        "Burmese": 0.8575
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-2025-08-07",
-      "name": "gpt-5-2025-08-07",
-      "developer": "OpenAI",
-      "scores": {
-        "Global MMLU Lite": 0.8895,
-        "Culturally Sensitive": 0.8913,
-        "Culturally Agnostic": 0.8878,
-        "Arabic": 0.8925,
-        "English": 0.8725,
-        "Bengali": 0.9,
-        "German": 0.91,
-        "French": 0.9075,
-        "Hindi": 0.865,
-        "Indonesian": 0.795,
-        "Italian": 0.9075,
-        "Japanese": 0.8875,
-        "Korean": 0.915,
-        "Portuguese": 0.8875,
-        "Spanish": 0.905,
-        "Swahili": 0.865,
-        "Yoruba": 0.9125,
-        "Chinese": 0.895,
-        "Burmese": 0.915
-      }
-    },
-    {
-      "model_id": "openai/o3-mini-2025-01-31",
-      "name": "o3-mini-2025-01-31",
-      "developer": "OpenAI",
-      "scores": {
-        "Global MMLU Lite": 0.78,
-        "Culturally Sensitive": 0.765,
-        "Culturally Agnostic": 0.795,
-        "Arabic": 0.7725,
-        "English": 0.8025,
-        "Bengali": 0.77,
-        "German": 0.7525,
-        "French": 0.74,
-        "Hindi": 0.7525,
-        "Indonesian": 0.7425,
-        "Italian": 0.8,
-        "Japanese": 0.81,
-        "Korean": 0.8075,
-        "Portuguese": 0.7975,
-        "Spanish": 0.775,
-        "Swahili": 0.765,
-        "Yoruba": 0.7725,
-        "Chinese": 0.8125,
-        "Burmese": 0.8075
-      }
-    },
-    {
-      "model_id": "openai/o4-mini-2025-04-16",
-      "name": "o4-mini-2025-04-16",
-      "developer": "OpenAI",
-      "scores": {
-        "Global MMLU Lite": 0.8705,
-        "Culturally Sensitive": 0.8503,
-        "Culturally Agnostic": 0.8906,
-        "Arabic": 0.865,
-        "English": 0.8675,
-        "Bengali": 0.8875,
-        "German": 0.8775,
-        "French": 0.87,
-        "Hindi": 0.87,
-        "Indonesian": 0.8675,
-        "Italian": 0.855,
-        "Japanese": 0.885,
-        "Korean": 0.88,
-        "Portuguese": 0.88,
-        "Spanish": 0.855,
-        "Swahili": 0.8525,
-        "Yoruba": 0.8525,
-        "Chinese": 0.89,
-        "Burmese": 0.8725
-      }
-    },
-    {
-      "model_id": "unknown/aya-expanse-32b",
-      "name": "aya-expanse-32b",
-      "developer": "unknown",
-      "scores": {
-        "Global MMLU Lite": 0.7353,
-        "Culturally Sensitive": 0.6891,
-        "Culturally Agnostic": 0.7815,
-        "Arabic": 0.7425,
-        "English": 0.7544,
-        "Bengali": 0.7343,
-        "German": 0.7425,
-        "French": 0.7325,
-        "Hindi": 0.7375,
-        "Indonesian": 0.7594,
-        "Italian": 0.7305,
-        "Japanese": 0.7419,
-        "Korean": 0.7525,
-        "Portuguese": 0.7544,
-        "Spanish": 0.7362,
-        "Swahili": 0.7071,
-        "Yoruba": 0.6942,
-        "Chinese": 0.743,
-        "Burmese": 0.7025
-      }
-    },
-    {
-      "model_id": "unknown/granite-4.0-h-small",
-      "name": "granite-4.0-h-small",
-      "developer": "unknown",
-      "scores": {
-        "Global MMLU Lite": 0.7503,
-        "Culturally Sensitive": 0.7182,
-        "Culturally Agnostic": 0.7826,
-        "Arabic": 0.7613,
-        "English": 0.77,
-        "Bengali": 0.7613,
-        "German": 0.755,
-        "French": 0.7594,
-        "Hindi": 0.7575,
-        "Indonesian": 0.7614,
-        "Italian": 0.7525,
-        "Japanese": 0.7406,
-        "Korean": 0.7525,
-        "Portuguese": 0.757,
-        "Spanish": 0.7638,
-        "Swahili": 0.7318,
-        "Yoruba": 0.6921,
-        "Chinese": 0.7475,
-        "Burmese": 0.7419
-      }
-    },
-    {
-      "model_id": "unknown/o4-mini-2025-04-16",
-      "name": "o4-mini-2025-04-16",
-      "developer": "unknown",
-      "scores": {
-        "Global MMLU Lite": 0.8705,
-        "Culturally Sensitive": 0.8503,
-        "Culturally Agnostic": 0.8906,
-        "Arabic": 0.865,
-        "English": 0.8675,
-        "Bengali": 0.8875,
-        "German": 0.8775,
-        "French": 0.87,
-        "Hindi": 0.87,
-        "Indonesian": 0.8675,
-        "Italian": 0.855,
-        "Japanese": 0.885,
-        "Korean": 0.88,
-        "Portuguese": 0.88,
-        "Spanish": 0.855,
-        "Swahili": 0.8525,
-        "Yoruba": 0.8525,
-        "Chinese": 0.89,
-        "Burmese": 0.8725
-      }
-    },
-    {
-      "model_id": "xai/grok-3-mini",
-      "name": "grok-3-mini",
-      "developer": "xAI",
-      "scores": {
-        "Global MMLU Lite": 0.673,
-        "Culturally Sensitive": 0.6717,
-        "Culturally Agnostic": 0.6743,
-        "Arabic": 0.755,
-        "English": 0.5075,
-        "Bengali": 0.7355,
-        "German": 0.6591,
-        "French": 0.485,
-        "Hindi": 0.56,
-        "Indonesian": 0.725,
-        "Italian": 0.696,
-        "Japanese": 0.6575,
-        "Korean": 0.7325,
-        "Portuguese": 0.6275,
-        "Spanish": 0.61,
-        "Swahili": 0.7625,
-        "Yoruba": 0.8296,
-        "Chinese": 0.5564,
-        "Burmese": 0.8693
-      }
-    },
-    {
-      "model_id": "xai/grok-4-0709",
-      "name": "grok-4-0709",
-      "developer": "xAI",
-      "scores": {
-        "Global MMLU Lite": 0.8881,
-        "Culturally Sensitive": 0.8862,
-        "Culturally Agnostic": 0.89,
-        "Arabic": 0.885,
-        "English": 0.905,
-        "Bengali": 0.8925,
-        "German": 0.8725,
-        "French": 0.875,
-        "Hindi": 0.8675,
-        "Indonesian": 0.89,
-        "Italian": 0.9025,
-        "Japanese": 0.87,
-        "Korean": 0.895,
-        "Portuguese": 0.8725,
-        "Spanish": 0.9075,
-        "Swahili": 0.91,
-        "Yoruba": 0.905,
-        "Chinese": 0.8525,
-        "Burmese": 0.9075
-      }
-    }
-  ]
-}

data/benchmarks/helm_capabilities.json

@@ -1,1026 +0,0 @@
-{
-  "benchmark_cards": {
-    "Omni-MATH": {
-      "benchmark_details": {
-        "name": "Omni-MATH",
-        "overview": "Omni-MATH is a benchmark designed to measure the mathematical reasoning capabilities of large language models at the Olympiad competition level. It comprises 4,428 challenging problems, categorized into over 33 sub-domains and more than 10 distinct difficulty levels. It was created because existing mathematical benchmarks are nearing saturation and are inadequate for assessing models on truly difficult tasks.",
-        "data_type": "text",
-        "domains": [
-          "math",
-          "olympiads"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "GSM8K",
-          "MATH"
-        ],
-        "resources": [
-          "https://arxiv.org/abs/2410.07985",
-          "https://huggingface.co/datasets/KbsdJames/Omni-MATH",
-          "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To provide a challenging benchmark for assessing and advancing the mathematical reasoning capabilities of large language models at the Olympiad and competition level, as existing benchmarks have become less challenging. It enables a nuanced analysis of performance across various mathematical disciplines and complexity levels.",
-        "audience": [
-          "Researchers evaluating large language models"
-        ],
-        "tasks": [
-          "Solving Olympiad-level mathematical problems",
-          "Solving competition-level mathematical problems",
-          "Rule-based evaluation on a filtered subset of problems (Omni-MATH-Rule)"
-        ],
-        "limitations": "Some problems are not suitable for reliable rule-based evaluation, leading to a filtered 'testable' set and an 'untestable' set.",
-        "out_of_scope_uses": [
-          "Not specified"
-        ]
-      },
-      "data": {
-        "source": "The data is sourced from contest pages, the AoPS Wiki, and the AoPS Forum. Initial raw counts from these sources were filtered to create the final dataset.",
-        "size": "4,428 problems. A subset of 2,821 problems (Omni-MATH-Rule) is suitable for rule-based evaluation, and a separate subset of 100 samples was used for meta-evaluation.",
-        "format": "JSON",
-        "annotation": "Problems underwent rigorous human annotation by a team including PhD and Master's students. Each problem was reviewed by two annotators for cross-validation, achieving high agreement rates."
-      },
-      "methodology": {
-        "methods": [
-          "Models are evaluated by generating solutions to the mathematical problems.",
-          "Evaluation is conducted using both model-based judgment (e.g., GPT-4o or the open-source Omni-Judge) and rule-based evaluation for a specific subset of problems (Omni-MATH-Rule)."
-        ],
-        "metrics": [
-          "Accuracy (Acc)"
-        ],
-        "calculation": "The overall score is the accuracy percentage, calculated as the number of correct solutions divided by the total number of problems.",
-        "interpretation": "Higher accuracy indicates stronger performance. The benchmark is considered challenging, as even advanced models achieve only moderate accuracy.",
-        "baseline_results": "Paper baselines (accuracy on full Omni-MATH / Omni-MATH-Rule subset): o1-mini (60.54% / 62.2%), o1-preview (52.55% / 51.7%), qwen2.5-MATH-72b-Instruct (36.20% / 35.7%), qwen2.5-MATH-7b-Instruct (33.22% / 32.3%), GPT-4o (30.49% / 29.2%), NuminaMATH-72b-cot (28.45% / 27.1%), DeepseekMATH-7b-RL (16.12% / 14.9%). EEE results: OLMo 2 32B Instruct March 2025 scored 0.161 (16.1%).",
-        "validation": "For the Omni-MATH-Rule subset, two PhD students annotated problems to determine suitability for rule-based evaluation, achieving 98% cross-validation accuracy. A meta-evaluation compared model-generated judgments against human gold-standard annotations on a 100-sample subset to assess reliability."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "Not specified",
-        "data_licensing": "Apache License 2.0",
-        "consent_procedures": "Not specified",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        },
-        {
-          "category": "Improper usage",
-          "description": [
-            "Improper usage occurs when a model is used for a purpose that it was not originally designed for."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "purpose_and_intended_users.out_of_scope_uses",
-        "ethical_and_legal_considerations.privacy_and_anonymity",
-        "ethical_and_legal_considerations.consent_procedures",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T13:34:44.331592",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    },
-    "WildBench": {
-      "benchmark_details": {
-        "name": "WildBench",
-        "overview": "WildBench is a benchmark that measures the performance of large language models on challenging, open-ended tasks derived from real-world user queries. It consists of 1,024 tasks selected from human-chatbot conversation logs. Its distinctive features include using a natural distribution of real user queries and employing automated evaluation with two novel metrics, WB-Reward and WB-Score, which utilize task-specific checklists and structured explanations.",
-        "data_type": "tabular, text",
-        "domains": [
-          "Info Seeking",
-          "Math & Data",
-          "Reasoning & Planning",
-          "Creative Tasks"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "AlpacaEval",
-          "ArenaHard",
-          "MT-bench",
-          "Chatbot Arena"
-        ],
-        "resources": [
-          "https://arxiv.org/abs/2406.04770",
-          "https://huggingface.co/datasets/allenai/WildBench",
-          "https://huggingface.co/spaces/allenai/WildBench",
-          "https://storage.googleapis.com/crfm-helm-public/capabilities/benchmark_output/releases/v1.15.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To provide a realistic, automated, and cost-effective evaluation framework for large language models that reflects their capabilities on challenging, real-world user tasks. It aims to be contamination-resilient and dynamically updated.",
-        "audience": [
-          "Researchers and practitioners evaluating large language models"
-        ],
-        "tasks": [
-          "Open-ended text generation in response to diverse user queries"
-        ],
-        "limitations": "The benchmark acknowledges a common issue in LLM-as-a-judge evaluations: bias towards longer outputs. It introduces a length-penalty method to mitigate this.",
-        "out_of_scope_uses": [
-          "Not specified"
-        ]
-      },
-      "data": {
-        "source": "The data is derived from real user-chatbot conversation logs collected by the AI2 WildChat project. Over one million logs were filtered to create the benchmark.",
-        "size": "The benchmark contains 1,024 tasks. A separate 'v2-hard' configuration contains 256 examples. The dataset falls into the size category of 1K to 10K examples.",
-        "format": "The data is stored in Parquet format.",
-        "annotation": "Human annotation was used for quality control. GPT-4-Turbo assisted by summarizing query intents to help human reviewers filter out nonsensical tasks. Human reviewers manually assessed tasks to ensure they were challenging, diverse, and that associated checklist questions were clear. The final set of tasks was retained after this process. Each example includes fine-grained annotations such as task types and checklists for evaluating response quality."
-      },
-      "methodology": {
-        "methods": [
-          "Models are evaluated automatically using LLM-as-a-judge, with GPT-4-turbo as the evaluator. The evaluation is not fine-tuning-based and appears to be zero-shot or prompted.",
-          "The evaluation uses length-penalized pairwise comparisons and individual scoring to prevent bias towards longer outputs."
-        ],
-        "metrics": [
-          "WB-Reward (for pairwise comparisons)",
-          "WB-Score (for individual scoring)",
-          "WB-Elo (for leaderboard ranking, merging WB-Reward and WB-Score)"
-        ],
-        "calculation": "WB-Reward compares a test model against three baseline models of varying performance, producing outcomes of 'much better', 'slightly better', 'tie', 'slightly worse', or 'much worse'. A length penalty converts 'slightly better/worse' to 'tie' if the winner's response is excessively longer. WB-Score evaluates individual model outputs. WB-Elo merges the pairwise comparisons from WB-Reward and WB-Score to perform Elo rating updates.",
-        "interpretation": "Higher scores on WB-Reward and WB-Score indicate better performance. Strong performance is validated by a high correlation with human-voted Elo ratings from Chatbot Arena, with WB-Reward achieving a Pearson correlation of 0.98 and WB-Score reaching 0.95.",
-        "baseline_results": "Paper baselines: The paper reports results for 40 LLMs but does not list specific scores. It notes that using Haiku as a baseline yields the best correlation with human ratings. EEE results: OLMo 2 32B Instruct March 2025 scored 0.7340 on WildBench.",
-        "validation": "The evaluation metrics are validated by their strong correlation with human-voted Elo ratings from Chatbot Arena on hard tasks. Checklist questions used in the evaluation were manually verified for clarity and relevance."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "Not specified",
-        "data_licensing": "The dataset is made available under a CC BY license, intended for research and educational use in accordance with AI2's Responsible Use Guidelines.",
-        "consent_procedures": "Not specified",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Data contamination",
-          "description": [
-            "Data contamination occurs when incorrect data is used for training. For example, data that is not aligned with model's purpose or data that is already set aside for other development tasks such as testing and evaluation."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-contamination.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "purpose_and_intended_users.out_of_scope_uses",
-        "ethical_and_legal_considerations.privacy_and_anonymity",
-        "ethical_and_legal_considerations.consent_procedures",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T13:56:24.159440",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    }
-  },
-  "models": [
-    {
-      "model_id": "allenai/OLMo-2-1124-7B-Instruct",
-      "name": "OLMo 2 7B Instruct November 2024",
-      "developer": "allenai",
-      "scores": {
-        "Mean score": 0.405,
-        "MMLU-Pro": 0.292,
-        "GPQA": 0.296,
-        "IFEval": 0.693,
-        "WildBench": 0.628,
-        "Omni-MATH": 0.116
-      }
-    },
-    {
-      "model_id": "allenai/OLMoE-1B-7B-0125-Instruct",
-      "name": "OLMoE 1B-7B Instruct January 2025",
-      "developer": "allenai",
-      "scores": {
-        "Mean score": 0.332,
-        "MMLU-Pro": 0.169,
-        "GPQA": 0.22,
-        "IFEval": 0.628,
-        "WildBench": 0.551,
-        "Omni-MATH": 0.093
-      }
-    },
-    {
-      "model_id": "allenai/olmo-2-0325-32b-instruct",
-      "name": "OLMo 2 32B Instruct March 2025",
-      "developer": "allenai",
-      "scores": {
-        "Mean score": 0.475,
-        "MMLU-Pro": 0.414,
-        "GPQA": 0.287,
-        "IFEval": 0.78,
-        "WildBench": 0.734,
-        "Omni-MATH": 0.161
-      }
-    },
-    {
-      "model_id": "allenai/olmo-2-1124-13b-instruct",
-      "name": "OLMo 2 13B Instruct November 2024",
-      "developer": "allenai",
-      "scores": {
-        "Mean score": 0.44,
-        "MMLU-Pro": 0.31,
-        "GPQA": 0.316,
-        "IFEval": 0.73,
-        "WildBench": 0.689,
-        "Omni-MATH": 0.156
-      }
-    },
-    {
-      "model_id": "amazon/nova-lite-v1:0",
-      "name": "Amazon Nova Lite",
-      "developer": "amazon",
-      "scores": {
-        "Mean score": 0.551,
-        "MMLU-Pro": 0.6,
-        "GPQA": 0.397,
-        "IFEval": 0.776,
-        "WildBench": 0.75,
-        "Omni-MATH": 0.233
-      }
-    },
-    {
-      "model_id": "amazon/nova-micro-v1:0",
-      "name": "Amazon Nova Micro",
-      "developer": "amazon",
-      "scores": {
-        "Mean score": 0.522,
-        "MMLU-Pro": 0.511,
-        "GPQA": 0.383,
-        "IFEval": 0.76,
-        "WildBench": 0.743,
-        "Omni-MATH": 0.214
-      }
-    },
-    {
-      "model_id": "amazon/nova-premier-v1:0",
-      "name": "Amazon Nova Premier",
-      "developer": "amazon",
-      "scores": {
-        "Mean score": 0.637,
-        "MMLU-Pro": 0.726,
-        "GPQA": 0.518,
-        "IFEval": 0.803,
-        "WildBench": 0.788,
-        "Omni-MATH": 0.35
-      }
-    },
-    {
-      "model_id": "amazon/nova-pro-v1:0",
-      "name": "Amazon Nova Pro",
-      "developer": "amazon",
-      "scores": {
-        "Mean score": 0.591,
-        "MMLU-Pro": 0.673,
-        "GPQA": 0.446,
-        "IFEval": 0.815,
-        "WildBench": 0.777,
-        "Omni-MATH": 0.242
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-5-haiku-20241022",
-      "name": "Claude 3.5 Haiku 20241022",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean score": 0.549,
-        "MMLU-Pro": 0.605,
-        "GPQA": 0.363,
-        "IFEval": 0.792,
-        "WildBench": 0.76,
-        "Omni-MATH": 0.224
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-5-sonnet-20241022",
-      "name": "Claude 3.5 Sonnet 20241022",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean score": 0.653,
-        "MMLU-Pro": 0.777,
-        "GPQA": 0.565,
-        "IFEval": 0.856,
-        "WildBench": 0.792,
-        "Omni-MATH": 0.276
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-7-sonnet-20250219",
-      "name": "claude-3-7-sonnet-20250219",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean score": 0.674,
-        "MMLU-Pro": 0.784,
-        "GPQA": 0.608,
-        "IFEval": 0.834,
-        "WildBench": 0.814,
-        "Omni-MATH": 0.33
-      }
-    },
-    {
-      "model_id": "anthropic/claude-opus-4-20250514",
-      "name": "Claude 4 Opus 20250514",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean score": 0.757,
-        "MMLU-Pro": 0.859,
-        "GPQA": 0.666,
-        "IFEval": 0.918,
-        "WildBench": 0.833,
-        "Omni-MATH": 0.511
-      }
-    },
-    {
-      "model_id": "anthropic/claude-opus-4-20250514-thinking-10k",
-      "name": "Claude 4 Opus 20250514, extended thinking",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean score": 0.78,
-        "MMLU-Pro": 0.875,
-        "GPQA": 0.709,
-        "IFEval": 0.849,
-        "WildBench": 0.852,
-        "Omni-MATH": 0.616
-      }
-    },
-    {
-      "model_id": "anthropic/claude-sonnet-4-20250514",
-      "name": "claude-sonnet-4-20250514",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean score": 0.733,
-        "MMLU-Pro": 0.843,
-        "GPQA": 0.643,
-        "IFEval": 0.839,
-        "WildBench": 0.825,
-        "Omni-MATH": 0.512
-      }
-    },
-    {
-      "model_id": "anthropic/claude-sonnet-4-20250514-thinking-10k",
-      "name": "Claude 4 Sonnet 20250514, extended thinking",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean score": 0.766,
-        "MMLU-Pro": 0.843,
-        "GPQA": 0.706,
-        "IFEval": 0.84,
-        "WildBench": 0.838,
-        "Omni-MATH": 0.602
-      }
-    },
-    {
-      "model_id": "deepseek-ai/deepseek-r1-0528",
-      "name": "DeepSeek-R1-0528",
-      "developer": "deepseek-ai",
-      "scores": {
-        "Mean score": 0.699,
-        "MMLU-Pro": 0.793,
-        "GPQA": 0.666,
-        "IFEval": 0.784,
-        "WildBench": 0.828,
-        "Omni-MATH": 0.424
-      }
-    },
-    {
-      "model_id": "deepseek-ai/deepseek-v3",
-      "name": "DeepSeek v3",
-      "developer": "deepseek-ai",
-      "scores": {
-        "Mean score": 0.665,
-        "MMLU-Pro": 0.723,
-        "GPQA": 0.538,
-        "IFEval": 0.832,
-        "WildBench": 0.831,
-        "Omni-MATH": 0.403
-      }
-    },
-    {
-      "model_id": "google/gemini-1.5-flash-002",
-      "name": "Gemini 1.5 Flash 002",
-      "developer": "Google",
-      "scores": {
-        "Mean score": 0.609,
-        "MMLU-Pro": 0.678,
-        "GPQA": 0.437,
-        "IFEval": 0.831,
-        "WildBench": 0.792,
-        "Omni-MATH": 0.305
-      }
-    },
-    {
-      "model_id": "google/gemini-1.5-pro-002",
-      "name": "Gemini 1.5 Pro 002",
-      "developer": "Google",
-      "scores": {
-        "Mean score": 0.657,
-        "MMLU-Pro": 0.737,
-        "GPQA": 0.534,
-        "IFEval": 0.837,
-        "WildBench": 0.813,
-        "Omni-MATH": 0.364
-      }
-    },
-    {
-      "model_id": "google/gemini-2.0-flash-001",
-      "name": "Gemini 2.0 Flash",
-      "developer": "Google",
-      "scores": {
-        "Mean score": 0.679,
-        "MMLU-Pro": 0.737,
-        "GPQA": 0.556,
-        "IFEval": 0.841,
-        "WildBench": 0.8,
-        "Omni-MATH": 0.459
-      }
-    },
-    {
-      "model_id": "google/gemini-2.0-flash-lite-preview-02-05",
-      "name": "Gemini 2.0 Flash Lite 02-05 preview",
-      "developer": "Google",
-      "scores": {
-        "Mean score": 0.642,
-        "MMLU-Pro": 0.72,
-        "GPQA": 0.5,
-        "IFEval": 0.824,
-        "WildBench": 0.79,
-        "Omni-MATH": 0.374
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-flash-lite",
-      "name": "Gemini 2.5 Flash-Lite",
-      "developer": "Google",
-      "scores": {
-        "Mean score": 0.591,
-        "MMLU-Pro": 0.537,
-        "GPQA": 0.309,
-        "IFEval": 0.81,
-        "WildBench": 0.818,
-        "Omni-MATH": 0.48
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-flash-preview-04-17",
-      "name": "Gemini 2.5 Flash 04-17 preview",
-      "developer": "Google",
-      "scores": {
-        "Mean score": 0.626,
-        "MMLU-Pro": 0.639,
-        "GPQA": 0.39,
-        "IFEval": 0.898,
-        "WildBench": 0.817,
-        "Omni-MATH": 0.384
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-pro-preview-03-25",
-      "name": "Gemini 2.5 Pro 03-25 preview",
-      "developer": "Google",
-      "scores": {
-        "Mean score": 0.745,
-        "MMLU-Pro": 0.863,
-        "GPQA": 0.749,
-        "IFEval": 0.84,
-        "WildBench": 0.857,
-        "Omni-MATH": 0.416
-      }
-    },
-    {
-      "model_id": "ibm/granite-3.3-8b-instruct",
-      "name": "IBM Granite 3.3 8B Instruct",
-      "developer": "ibm",
-      "scores": {
-        "Mean score": 0.463,
-        "MMLU-Pro": 0.343,
-        "GPQA": 0.325,
-        "IFEval": 0.729,
-        "WildBench": 0.741,
-        "Omni-MATH": 0.176
-      }
-    },
-    {
-      "model_id": "marin-community/marin-8b-instruct",
-      "name": "Marin 8B Instruct",
-      "developer": "marin-community",
-      "scores": {
-        "Mean score": 0.325,
-        "MMLU-Pro": 0.188,
-        "GPQA": 0.168,
-        "IFEval": 0.632,
-        "WildBench": 0.477,
-        "Omni-MATH": 0.16
-      }
-    },
-    {
-      "model_id": "meta/llama-3.1-405b-instruct-turbo",
-      "name": "Llama 3.1 Instruct Turbo 405B",
-      "developer": "Meta",
-      "scores": {
-        "Mean score": 0.618,
-        "MMLU-Pro": 0.723,
-        "GPQA": 0.522,
-        "IFEval": 0.811,
-        "WildBench": 0.783,
-        "Omni-MATH": 0.249
-      }
-    },
-    {
-      "model_id": "meta/llama-3.1-70b-instruct-turbo",
-      "name": "Llama 3.1 Instruct Turbo 70B",
-      "developer": "Meta",
-      "scores": {
-        "Mean score": 0.574,
-        "MMLU-Pro": 0.653,
-        "GPQA": 0.426,
-        "IFEval": 0.821,
-        "WildBench": 0.758,
-        "Omni-MATH": 0.21
-      }
-    },
-    {
-      "model_id": "meta/llama-3.1-8b-instruct-turbo",
-      "name": "Llama 3.1 Instruct Turbo 8B",
-      "developer": "Meta",
-      "scores": {
-        "Mean score": 0.444,
-        "MMLU-Pro": 0.406,
-        "GPQA": 0.247,
-        "IFEval": 0.743,
-        "WildBench": 0.686,
-        "Omni-MATH": 0.137
-      }
-    },
-    {
-      "model_id": "meta/llama-4-maverick-17b-128e-instruct-fp8",
-      "name": "Llama 4 Maverick 17Bx128E Instruct FP8",
-      "developer": "Meta",
-      "scores": {
-        "Mean score": 0.718,
-        "MMLU-Pro": 0.81,
-        "GPQA": 0.65,
-        "IFEval": 0.908,
-        "WildBench": 0.8,
-        "Omni-MATH": 0.422
-      }
-    },
-    {
-      "model_id": "meta/llama-4-scout-17b-16e-instruct",
-      "name": "Llama 4 Scout 17Bx16E Instruct",
-      "developer": "Meta",
-      "scores": {
-        "Mean score": 0.644,
-        "MMLU-Pro": 0.742,
-        "GPQA": 0.507,
-        "IFEval": 0.818,
-        "WildBench": 0.779,
-        "Omni-MATH": 0.373
-      }
-    },
-    {
-      "model_id": "mistralai/Mixtral-8x22B-Instruct-v0.1",
-      "name": "Mixtral-8x22B-Instruct-v0.1",
-      "developer": "mistralai",
-      "scores": {
-        "Mean score": 0.478,
-        "MMLU-Pro": 0.46,
-        "GPQA": 0.334,
-        "IFEval": 0.724,
-        "WildBench": 0.711,
-        "Omni-MATH": 0.163
-      }
-    },
-    {
-      "model_id": "mistralai/Mixtral-8x7B-Instruct-v0.1",
-      "name": "Mixtral-8x7B-Instruct-v0.1",
-      "developer": "mistralai",
-      "scores": {
-        "Mean score": 0.397,
-        "MMLU-Pro": 0.335,
-        "GPQA": 0.296,
-        "IFEval": 0.575,
-        "WildBench": 0.673,
-        "Omni-MATH": 0.105
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-7b-instruct-v0.3",
-      "name": "Mistral Instruct v0.3 7B",
-      "developer": "mistralai",
-      "scores": {
-        "Mean score": 0.376,
-        "MMLU-Pro": 0.277,
-        "GPQA": 0.303,
-        "IFEval": 0.567,
-        "WildBench": 0.66,
-        "Omni-MATH": 0.072
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-large-2411",
-      "name": "Mistral Large 2411",
-      "developer": "mistralai",
-      "scores": {
-        "Mean score": 0.598,
-        "MMLU-Pro": 0.599,
-        "GPQA": 0.435,
-        "IFEval": 0.876,
-        "WildBench": 0.801,
-        "Omni-MATH": 0.281
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-small-2503",
-      "name": "mistral-small-2503",
-      "developer": "mistralai",
-      "scores": {
-        "Mean score": 0.558,
-        "MMLU-Pro": 0.61,
-        "GPQA": 0.392,
-        "IFEval": 0.75,
-        "WildBench": 0.788,
-        "Omni-MATH": 0.248
-      }
-    },
-    {
-      "model_id": "moonshotai/kimi-k2-instruct",
-      "name": "Kimi K2 Instruct",
-      "developer": "moonshotai",
-      "scores": {
-        "Mean score": 0.768,
-        "MMLU-Pro": 0.819,
-        "GPQA": 0.652,
-        "IFEval": 0.85,
-        "WildBench": 0.862,
-        "Omni-MATH": 0.654
-      }
-    },
-    {
-      "model_id": "openai/gpt-4.1-2025-04-14",
-      "name": "gpt-4.1-2025-04-14",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.727,
-        "MMLU-Pro": 0.811,
-        "GPQA": 0.659,
-        "IFEval": 0.838,
-        "WildBench": 0.854,
-        "Omni-MATH": 0.471
-      }
-    },
-    {
-      "model_id": "openai/gpt-4.1-mini-2025-04-14",
-      "name": "GPT-4.1 mini 2025-04-14",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.726,
-        "MMLU-Pro": 0.783,
-        "GPQA": 0.614,
-        "IFEval": 0.904,
-        "WildBench": 0.838,
-        "Omni-MATH": 0.491
-      }
-    },
-    {
-      "model_id": "openai/gpt-4.1-nano-2025-04-14",
-      "name": "GPT-4.1 nano 2025-04-14",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.616,
-        "MMLU-Pro": 0.55,
-        "GPQA": 0.507,
-        "IFEval": 0.843,
-        "WildBench": 0.811,
-        "Omni-MATH": 0.367
-      }
-    },
-    {
-      "model_id": "openai/gpt-4o-2024-11-20",
-      "name": "GPT-4o 2024-11-20",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.634,
-        "MMLU-Pro": 0.713,
-        "GPQA": 0.52,
-        "IFEval": 0.817,
-        "WildBench": 0.828,
-        "Omni-MATH": 0.293
-      }
-    },
-    {
-      "model_id": "openai/gpt-4o-mini-2024-07-18",
-      "name": "GPT-4o mini 2024-07-18",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.565,
-        "MMLU-Pro": 0.603,
-        "GPQA": 0.368,
-        "IFEval": 0.782,
-        "WildBench": 0.791,
-        "Omni-MATH": 0.28
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-2025-08-07",
-      "name": "gpt-5-2025-08-07",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.807,
-        "MMLU-Pro": 0.863,
-        "GPQA": 0.791,
-        "IFEval": 0.875,
-        "WildBench": 0.857,
-        "Omni-MATH": 0.647
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-mini-2025-08-07",
-      "name": "GPT-5 mini 2025-08-07",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.819,
-        "MMLU-Pro": 0.835,
-        "GPQA": 0.756,
-        "IFEval": 0.927,
-        "WildBench": 0.855,
-        "Omni-MATH": 0.722
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-nano-2025-08-07",
-      "name": "GPT-5 nano 2025-08-07",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.748,
-        "MMLU-Pro": 0.778,
-        "GPQA": 0.679,
-        "IFEval": 0.932,
-        "WildBench": 0.806,
-        "Omni-MATH": 0.547
-      }
-    },
-    {
-      "model_id": "openai/gpt-oss-120b",
-      "name": "GPT-OSS-120B",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.77,
-        "MMLU-Pro": 0.795,
-        "GPQA": 0.684,
-        "IFEval": 0.836,
-        "WildBench": 0.845,
-        "Omni-MATH": 0.688
-      }
-    },
-    {
-      "model_id": "openai/gpt-oss-20b",
-      "name": "GPT-OSS-20B",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.674,
-        "MMLU-Pro": 0.74,
-        "GPQA": 0.594,
-        "IFEval": 0.732,
-        "WildBench": 0.737,
-        "Omni-MATH": 0.565
-      }
-    },
-    {
-      "model_id": "openai/o3-2025-04-16",
-      "name": "o3-2025-04-16",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.811,
-        "MMLU-Pro": 0.859,
-        "GPQA": 0.753,
-        "IFEval": 0.869,
-        "WildBench": 0.861,
-        "Omni-MATH": 0.714
-      }
-    },
-    {
-      "model_id": "openai/o4-mini-2025-04-16",
-      "name": "o4-mini-2025-04-16",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean score": 0.812,
-        "MMLU-Pro": 0.82,
-        "GPQA": 0.735,
-        "IFEval": 0.929,
-        "WildBench": 0.854,
-        "Omni-MATH": 0.72
-      }
-    },
-    {
-      "model_id": "qwen/qwen2.5-72b-instruct-turbo",
-      "name": "Qwen2.5 Instruct Turbo 72B",
-      "developer": "qwen",
-      "scores": {
-        "Mean score": 0.599,
-        "MMLU-Pro": 0.631,
-        "GPQA": 0.426,
-        "IFEval": 0.806,
-        "WildBench": 0.802,
-        "Omni-MATH": 0.33
-      }
-    },
-    {
-      "model_id": "qwen/qwen2.5-7b-instruct-turbo",
-      "name": "Qwen2.5 Instruct Turbo 7B",
-      "developer": "qwen",
-      "scores": {
-        "Mean score": 0.529,
-        "MMLU-Pro": 0.539,
-        "GPQA": 0.341,
-        "IFEval": 0.741,
-        "WildBench": 0.731,
-        "Omni-MATH": 0.294
-      }
-    },
-    {
-      "model_id": "qwen/qwen3-235b-a22b-fp8-tput",
-      "name": "Qwen3 235B A22B FP8 Throughput",
-      "developer": "qwen",
-      "scores": {
-        "Mean score": 0.726,
-        "MMLU-Pro": 0.817,
-        "GPQA": 0.623,
-        "IFEval": 0.816,
-        "WildBench": 0.828,
-        "Omni-MATH": 0.548
-      }
-    },
-    {
-      "model_id": "qwen/qwen3-235b-a22b-instruct-2507-fp8",
-      "name": "Qwen3 235B A22B Instruct 2507 FP8",
-      "developer": "qwen",
-      "scores": {
-        "Mean score": 0.798,
-        "MMLU-Pro": 0.844,
-        "GPQA": 0.726,
-        "IFEval": 0.835,
-        "WildBench": 0.866,
-        "Omni-MATH": 0.718
-      }
-    },
-    {
-      "model_id": "writer/palmyra-fin",
-      "name": "Palmyra Fin",
-      "developer": "writer",
-      "scores": {
-        "Mean score": 0.577,
-        "MMLU-Pro": 0.591,
-        "GPQA": 0.422,
-        "IFEval": 0.793,
-        "WildBench": 0.783,
-        "Omni-MATH": 0.295
-      }
-    },
-    {
-      "model_id": "writer/palmyra-med",
-      "name": "Palmyra Med",
-      "developer": "writer",
-      "scores": {
-        "Mean score": 0.476,
-        "MMLU-Pro": 0.411,
-        "GPQA": 0.368,
-        "IFEval": 0.767,
-        "WildBench": 0.676,
-        "Omni-MATH": 0.156
-      }
-    },
-    {
-      "model_id": "writer/palmyra-x-004",
-      "name": "Palmyra-X-004",
-      "developer": "writer",
-      "scores": {
-        "Mean score": 0.609,
-        "MMLU-Pro": 0.657,
-        "GPQA": 0.395,
-        "IFEval": 0.872,
-        "WildBench": 0.802,
-        "Omni-MATH": 0.32
-      }
-    },
-    {
-      "model_id": "writer/palmyra-x5",
-      "name": "Palmyra X5",
-      "developer": "writer",
-      "scores": {
-        "Mean score": 0.696,
-        "MMLU-Pro": 0.804,
-        "GPQA": 0.661,
-        "IFEval": 0.823,
-        "WildBench": 0.78,
-        "Omni-MATH": 0.414
-      }
-    },
-    {
-      "model_id": "xai/grok-3-beta",
-      "name": "Grok 3 Beta",
-      "developer": "xAI",
-      "scores": {
-        "Mean score": 0.727,
-        "MMLU-Pro": 0.788,
-        "GPQA": 0.65,
-        "IFEval": 0.884,
-        "WildBench": 0.849,
-        "Omni-MATH": 0.464
-      }
-    },
-    {
-      "model_id": "xai/grok-3-mini-beta",
-      "name": "Grok 3 mini Beta",
-      "developer": "xAI",
-      "scores": {
-        "Mean score": 0.679,
-        "MMLU-Pro": 0.799,
-        "GPQA": 0.675,
-        "IFEval": 0.951,
-        "WildBench": 0.651,
-        "Omni-MATH": 0.318
-      }
-    },
-    {
-      "model_id": "xai/grok-4-0709",
-      "name": "grok-4-0709",
-      "developer": "xAI",
-      "scores": {
-        "Mean score": 0.785,
-        "MMLU-Pro": 0.851,
-        "GPQA": 0.726,
-        "IFEval": 0.949,
-        "WildBench": 0.797,
-        "Omni-MATH": 0.603
-      }
-    },
-    {
-      "model_id": "zai-org/glm-4.5-air-fp8",
-      "name": "GLM-4.5-Air-FP8",
-      "developer": "zai-org",
-      "scores": {
-        "Mean score": 0.67,
-        "MMLU-Pro": 0.762,
-        "GPQA": 0.594,
-        "IFEval": 0.812,
-        "WildBench": 0.789,
-        "Omni-MATH": 0.391
-      }
-    }
-  ]
-}

data/benchmarks/helm_classic.json

@@ -1,2030 +0,0 @@
-{
-  "benchmark_cards": {
-    "BoolQ": {
-      "benchmark_details": {
-        "name": "BoolQ",
-        "overview": "BoolQ is a benchmark that measures a model's ability to answer naturally occurring yes/no questions, framed as a reading comprehension task. The questions are generated in unprompted and unconstrained settings, often querying complex, non-factoid information and requiring difficult entailment-like inference. The dataset consists of a single task: answering yes/no questions given a supporting passage.",
-        "data_type": "text",
-        "domains": [
-          "natural language understanding",
-          "reading comprehension",
-          "natural language inference"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "MultiNLI",
-          "SNLI",
-          "QNLI",
-          "SQuAD 2.0",
-          "Natural Questions (NQ)",
-          "QQP",
-          "MS MARCO",
-          "RACE",
-          "bAbI stories"
-        ],
-        "resources": [
-          "https://arxiv.org/abs/1905.10044",
-          "https://huggingface.co/datasets/google/boolq",
-          "https://goo.gl/boolq",
-          "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To test models on their ability to answer naturally occurring yes/no questions, which are challenging and require complex inferential abilities beyond surface-level reasoning.",
-        "audience": [
-          "Researchers in natural language understanding and reading comprehension"
-        ],
-        "tasks": [
-          "Yes/no question answering",
-          "Text-pair classification"
-        ],
-        "limitations": "Annotation involved some errors and ambiguous cases. The use of singly-annotated examples is a trade-off for dataset size. Potential concerns about annotation artifacts are acknowledged.",
-        "out_of_scope_uses": "The paper does not explicitly state what the benchmark is not designed for."
-      },
-      "data": {
-        "source": "The data consists of naturally occurring yes/no questions authored by people who were not prompted to write specific question types and did not know the answers. The passages are excerpts from sources like Wikipedia.",
-        "size": "15,942 examples total, with 9,427 in the train split and 3,270 in the validation split. The dataset size category is between 10,000 and 100,000 examples.",
-        "format": "parquet",
-        "annotation": "Questions were answered by human annotators. A quality check on a subset showed the main annotation process achieved 90% accuracy against a gold-standard set labeled by three authors. The training, development, and test sets use singly-annotated examples."
-      },
-      "methodology": {
-        "methods": [
-          "Models are evaluated by fine-tuning on the BoolQ training set, potentially after transfer learning from other datasets or unsupervised pre-training. Zero-shot or direct use of pre-trained models without fine-tuning did not outperform the majority baseline.",
-          "The task requires providing a yes/no (boolean) answer to a question based on a given passage."
-        ],
-        "metrics": [
-          "Accuracy"
-        ],
-        "calculation": "The overall score is the accuracy percentage on the test set.",
-        "interpretation": "Higher accuracy indicates better performance. Human accuracy is 90%, and the majority baseline is approximately 62%.",
-        "baseline_results": "Paper baselines: Majority baseline: 62.17% dev, 62.31% test; Recurrent model baseline: 69.6%; Best model (BERT large pre-trained on MultiNLI then fine-tuned on BoolQ): 80.4% accuracy; Human accuracy: 90%. EEE results: Anthropic-LM v4-s3 52B: 81.5%.",
-        "validation": "Quality assurance involved author-led gold-standard annotation on a subset, showing 90% agreement. The development set was used for model selection, such as choosing the best model from five seeds based on its performance."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "Not specified",
-        "data_licensing": "cc-by-sa-3.0",
-        "consent_procedures": "Not specified",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Uncertain data provenance",
-          "description": [
-            "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "ethical_and_legal_considerations.privacy_and_anonymity",
-        "ethical_and_legal_considerations.consent_procedures",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T15:08:51.830946",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    },
-    "CNN/DailyMail": {
-      "benchmark_details": {
-        "name": "CNN/DailyMail",
-        "overview": "CNN/DailyMail is a benchmark for evaluating abstractive and extractive summarization models using news articles. It contains over 300,000 unique articles written by journalists from CNN and the Daily Mail. The dataset was originally created for machine reading and question answering, but later versions were restructured specifically for summarization tasks.",
-        "data_type": "text",
-        "domains": [
-          "summarization",
-          "journalism",
-          "news media"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": "No facts provided about similar benchmarks.",
-        "resources": [
-          "https://huggingface.co/datasets/abisee/cnn_dailymail",
-          "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To help develop models that can summarize long paragraphs of text into one or two sentences, aiding in the efficient presentation of information from large quantities of text.",
-        "audience": [
-          "NLP researchers",
-          "Summarization model developers"
-        ],
-        "tasks": [
-          "Summarization"
-        ],
-        "limitations": "News articles often place important information in the first third, which may affect summarization. A manual study found 25% of samples in an earlier version were difficult for humans due to ambiguity and coreference errors. Also, machine-generated summaries may differ in truth values from the original articles.",
-        "out_of_scope_uses": "No facts provided about out-of-scope uses."
-      },
-      "data": {
-        "source": "The dataset consists of news articles and highlight sentences written by journalists at CNN and the Daily Mail. The CNN articles were collected from April 2007 to April 2015, and the Daily Mail articles from June 2010 to April 2015, sourced from archives on the Wayback Machine.",
-        "size": "Over 300,000 unique articles, with 287,113 training examples, 13,368 validation examples, and 11,490 test examples.",
-        "format": "parquet",
-        "annotation": "The dataset does not contain additional annotations. The highlights are the original summaries written by the article authors and are used as the target for summarization."
-      },
-      "methodology": {
-        "methods": [
-          "Models generate a summary for a given news article, which is then compared to the author-written highlights."
-        ],
-        "metrics": [
-          "ROUGE-2"
-        ],
-        "calculation": "The ROUGE-2 score measures the overlap of bigrams between the generated summary and the reference highlights.",
-        "interpretation": "Higher scores indicate better performance, as they reflect greater overlap with the reference summaries.",
-        "baseline_results": "Paper baseline (Zhong et al., 2020): ROUGE-1 score of 44.41 for an extractive summarization model. Evaluation suite result (Anthropic-LM v4-s3 52B): ROUGE-2 score of 0.154.",
-        "validation": "No facts provided about validation procedures."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "The dataset (version 3.0.0) is not anonymized, meaning individuals' names are present in the text.",
-        "data_licensing": "Apache License 2.0",
-        "consent_procedures": "Not specified",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Data contamination",
-          "description": [
-            "Data contamination occurs when incorrect data is used for training. For example, data that is not aligned with model's purpose or data that is already set aside for other development tasks such as testing and evaluation."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-contamination.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "ethical_and_legal_considerations.consent_procedures",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T15:15:47.316103",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    },
-    "CivilComments": {
-      "benchmark_details": {
-        "name": "CivilComments",
-        "overview": "CivilComments is a benchmark designed to measure unintended identity-based bias in toxicity classification models. It uses a large, real-world dataset of online comments from the Civil Comments platform, extended with crowd-sourced annotations for toxicity and demographic identity references. This provides a nuanced evaluation of bias beyond synthetic datasets.",
-        "data_type": "tabular, text",
-        "domains": [
-          "machine learning fairness",
-          "bias measurement",
-          "toxic comment classification",
-          "text classification"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": "The paper does not name other specific benchmark datasets, only referencing prior work using synthetic test sets.",
-        "resources": [
-          "https://arxiv.org/abs/1903.04561",
-          "https://huggingface.co/datasets/google/civil_comments",
-          "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To evaluate unintended identity-based bias in toxicity classification models using real data and nuanced metrics.",
-        "audience": [
-          "Machine learning researchers and practitioners working on fairness, bias measurement, and mitigation, particularly in toxic comment classification."
-        ],
-        "tasks": [
-          "Binary toxicity classification (toxic vs. non-toxic)",
-          "Analysis of performance across identity subgroups"
-        ],
-        "limitations": "The labeled set of identities is not comprehensive and does not provide universal coverage, representing a balance between coverage, annotator accuracy, and example count. The real-world data is potentially noisier than synthetic alternatives.",
-        "out_of_scope_uses": [
-          "Developing effective strategies for choosing optimal thresholds to minimize bias"
-        ]
-      },
-      "data": {
-        "source": "The data consists of online comments sourced from the Civil Comments platform, a commenting plugin for independent English-language news sites. The comments were publicly posted between 2015 and 2017.",
-        "size": "The dataset contains approximately 1.8 million comments for training, with separate validation and test sets of approximately 97,320 examples each. All comments were labeled for toxicity, and a subset of 450,000 comments was additionally labeled for identity references.",
-        "format": "parquet",
-        "annotation": "Labeling was performed by crowd raters. Toxicity labels were applied using guidelines consistent with the Perspective API. For the identity-labeled subset, raters were shown comments and selected referenced identities (e.g., genders, races, ethnicities) from a provided list. Some comments for identity labeling were pre-selected by models to increase the frequency of identity content."
-      },
-      "methodology": {
-        "methods": [
-          "Models are evaluated by applying a suite of bias metrics to their predictions on the test set. The original paper demonstrates this using publicly accessible toxicity classifiers on the dataset."
-        ],
-        "metrics": [
-          "Subgroup AUC",
-          "BPSN AUC",
-          "BNSP AUC",
-          "Negative Average Equality Gap (AEG)",
-          "Positive Average Equality Gap (AEG)"
-        ],
-        "calculation": "The evaluation calculates five metrics for each identity subgroup to provide a multi-faceted view of bias. There is no single aggregated overall score.",
-        "interpretation": "For the AUC metrics (Subgroup, BPSN, BNSP), higher values indicate better separability (fewer mis-orderings). For the Average Equality Gaps (Negative and Positive), lower values indicate better separability (more similar score distributions).",
-        "baseline_results": "Paper baselines: Results for TOXICITY@1 and TOXICITY@6 from the Perspective API are reported, showing their Subgroup AUC, BPSN AUC, BNSP AUC, Negative AEG, and Positive AEG on a synthetic dataset for the lowest performing 20 subgroups. They are also compared on short comments within the human-labeled dataset for specific identities. EEE results: Anthropic-LM v4-s3 52B scored 0.6100 on the CivilComments metric.",
-        "validation": "The evaluation assumes the human-provided labels are reliable. The identity labeling set was designed to balance coverage, crowd rater accuracy, and ensure sufficient examples per identity for meaningful results."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "The paper does not discuss how personally identifiable information (PII) in the online comments was handled or if data was anonymized.",
-        "data_licensing": "Creative Commons Zero v1.0 Universal",
-        "consent_procedures": "The paper does not describe compensation for crowdworkers or the specific platform used for annotation.",
-        "compliance_with_regulations": "The paper does not mention IRB approval, GDPR compliance, or any other ethical review process."
-      },
-      "possible_risks": [
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Uncertain data provenance",
-          "description": [
-            "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        },
-        {
-          "category": "Output bias",
-          "description": [
-            "Generated content might unfairly represent certain groups or individuals."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/output-bias.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [],
-      "card_info": {
-        "created_at": "2026-03-17T12:38:43.250822",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    },
-    "HellaSwag": {
-      "benchmark_details": {
-        "name": "HellaSwag",
-        "overview": "HellaSwag is a benchmark designed to measure commonsense natural language inference by testing a model's ability to select the most plausible follow-up event from four multiple-choice options. It is adversarially constructed to be challenging for state-of-the-art models, using a method called Adversarial Filtering to create difficult wrong answers that are obvious to humans but often misclassified by models.",
-        "data_type": "text",
-        "domains": [
-          "commonsense reasoning",
-          "natural language inference"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "SWAG",
-          "SNLI"
-        ],
-        "resources": [
-          "https://rowanzellers.com/hellaswag",
-          "https://arxiv.org/abs/1905.07830",
-          "https://huggingface.co/datasets/Rowan/hellaswag",
-          "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To create a challenge dataset that reveals the difficulty of commonsense inference for state-of-the-art models, demonstrating their lack of robustness and reliance on dataset biases rather than genuine reasoning. It aims to evaluate a model's ability to select the most plausible continuation of a given event description.",
-        "audience": [
-          "NLP researchers"
-        ],
-        "tasks": [
-          "Four-way multiple-choice selection for event continuation",
-          "Commonsense inference"
-        ],
-        "limitations": "The adversarial filtering process used to create the dataset, while effective at making it difficult for models, may also select examples where the ground truth answer is not the one preferred by human annotators, necessitating manual filtering to retain the best examples.",
-        "out_of_scope_uses": [
-          "Not specified"
-        ]
-      },
-      "data": {
-        "source": "Contexts are sourced from WikiHow instructional articles and ActivityNet video descriptions. Incorrect answer choices are generated by machines and then adversarially filtered.",
-        "size": "The dataset contains 70,000 examples in total, with 5,001 in-domain validation examples and 5,000 zero-shot validation examples. The training set comprises 39,905 examples.",
-        "format": "Parquet",
-        "annotation": "Human crowd workers on Amazon Mechanical Turk validated the endings. They were presented with a context and six endings (one true, five machine-generated) and rated their plausibility. The process involved iterative filtering and replacement of unrealistic endings. Worker quality was ensured via an autograded test and fair pay. A gold standard check by three authors on a random sample showed 90% agreement with crowd annotations."
-      },
-      "methodology": {
-        "methods": [
-          "Models are evaluated via fine-tuning on the dataset.",
-          "The benchmark also includes zero-shot evaluation on held-out categories."
-        ],
-        "metrics": [
-          "HellaSwag accuracy"
-        ],
-        "calculation": "The overall score is the accuracy percentage on the full validation or test sets. Performance is also broken down by subsets, such as in-domain versus zero-shot and by data source.",
-        "interpretation": "Higher scores indicate better performance. Human performance is over 95%, which is considered strong. Model performance below 50% is reported, indicating a struggle, with a gap of over 45% from human performance on in-domain data.",
-        "baseline_results": "Paper baselines: BERT-Large achieves 47.3% accuracy overall. ESIM + ELMo gets 33.3% accuracy. A BERT-Base model with a frozen encoder and an added LSTM performs 4.3% worse than fine-tuned BERT-Base. Evaluation suite results: Anthropic-LM v4-s3 52B achieves 0.807 (80.7%) accuracy.",
-        "validation": "Human validation involved giving five crowd workers the same multiple-choice task and combining their answers via majority vote to establish a human performance baseline. The adversarial filtering process used iterative human ratings to ensure wrong answers were implausible."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "Not specified",
-        "data_licensing": "Not specified",
-        "consent_procedures": "Crowdworkers on Amazon Mechanical Turk participated voluntarily and were compensated, with pay described as fair. A qualification task was used to filter workers, and those who consistently preferred generated endings over real ones were disqualified.",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        },
-        {
-          "category": "Improper usage",
-          "description": [
-            "Improper usage occurs when a model is used for a purpose that it was not originally designed for."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html"
-        }
-      ],
-      "flagged_fields": {
-        "baseline_results": "[Possible Hallucination], no supporting evidence found in source material"
-      },
-      "missing_fields": [
-        "purpose_and_intended_users.out_of_scope_uses",
-        "ethical_and_legal_considerations.privacy_and_anonymity",
-        "ethical_and_legal_considerations.data_licensing",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T15:47:07.561060",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    },
-    "QuAC": {
-      "benchmark_details": {
-        "name": "QuAC",
-        "overview": "QuAC (Question Answering in Context) is a benchmark that measures a model's ability to answer questions within an information-seeking dialogue. It contains 14,000 dialogues comprising 100,000 question-answer pairs. The dataset is distinctive because questions are often open-ended, context-dependent, unanswerable, or only meaningful within the dialog flow, presenting challenges not found in standard machine comprehension datasets.",
-        "data_type": "text",
-        "domains": [
-          "question answering",
-          "dialogue modeling",
-          "text generation"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "SQuAD"
-        ],
-        "resources": [
-          "http://quac.ai",
-          "https://arxiv.org/abs/1808.07036",
-          "https://huggingface.co/datasets/allenai/quac",
-          "https://storage.googleapis.com/crfm-helm-public/benchmark_output/releases/v0.4.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To enable models to learn from and participate in information-seeking dialog, handling context-dependent, elliptical, and sometimes unanswerable questions.",
-        "audience": [
-          "Not specified"
-        ],
-        "tasks": [
-          "Extractive question answering",
-          "Text generation",
-          "Fill mask"
-        ],
-        "limitations": "Some questions have lower quality annotations; the dataset filters out the noisiest ~10% of annotations where human F1 is below 40. Questions can be open-ended, unanswerable, or only meaningful within the dialog context, posing inherent challenges.",
-        "out_of_scope_uses": [
-          "Not specified"
-        ]
-      },
-      "data": {
-        "source": "The data is crowdsourced via an interactive dialog between two crowd workers: one acting as a student asking questions to learn about a hidden Wikipedia text, and the other acting as a teacher who answers using short excerpts from that text. The source data comes from Wikipedia.",
-        "size": "The dataset contains 98,407 question-answer pairs from 13,594 dialogs, based on 8,854 unique sections from 3,611 unique Wikipedia articles. The training set has 83,568 questions (11,567 dialogs), the validation set has 7,354 questions (1,000 dialogs), and the test set has 7,353 questions (1,002 dialogs). The dataset size is between 10,000 and 100,000 examples.",
-        "format": "Each dialog is a sequence of question-answer pairs centered around a Wikipedia section. The teacher's response includes a text span, a 'yes/no' indication, a 'no answer' indication, and an encouragement for follow-up questions.",
-        "annotation": "Questions are answered by a teacher selecting short excerpts (spans) from the Wikipedia text. The training set has one reference answer per question, while the validation and test sets each have five reference answers per question to improve evaluation reliability. For evaluation, questions with a human F1 score lower than 40 are not used, as manual inspection revealed lower quality below this threshold."
-      },
-      "methodology": {
-        "methods": [
-          "Models predict a text span to answer a question about a Wikipedia section, given a dialog history of previous questions and answers.",
-          "The evaluation uses a reading comprehension architecture extended to model dialog context."
-        ],
-        "metrics": [
-          "Word-level F1"
-        ],
-        "calculation": "Precision and recall are computed over overlapping words after removing stopwords. For 'no answer' questions, F1 is 1 if correctly predicted and 0 otherwise. The maximum F1 among all references is computed for each question.",
-        "interpretation": "Higher F1 scores indicate better performance. The best model underperforms humans by 20 F1, indicating significant room for improvement.",
-        "baseline_results": "Paper baselines: The best model underperforms humans by 20 F1, but specific model names and scores are not provided. EEE results: Anthropic-LM v4-s3 52B achieves an F1 score of 0.431.",
-        "validation": "Quality assurance includes using multiple references for development and test questions, filtering out questions with low human F1 scores, and manual inspection of low-quality annotations."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "Not specified",
-        "data_licensing": "MIT License",
-        "consent_procedures": "Dialogs were created by two crowd workers, but the specific compensation or platform details are not provided in the paper.",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Uncertain data provenance",
-          "description": [
-            "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "purpose_and_intended_users.audience",
-        "purpose_and_intended_users.out_of_scope_uses",
-        "ethical_and_legal_considerations.privacy_and_anonymity",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T13:45:24.009083",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    }
-  },
-  "models": [
-    {
-      "model_id": "Anthropic-LM-v4-s3-52B",
-      "name": "Anthropic-LM v4-s3 52B",
-      "developer": "unknown",
-      "scores": {
-        "Mean win rate": 0.78,
-        "MMLU": 0.481,
-        "BoolQ": 0.815,
-        "NarrativeQA": 0.728,
-        "NaturalQuestions (open-book)": 0.686,
-        "QuAC": 0.431,
-        "HellaSwag": 0.807,
-        "OpenbookQA": 0.558,
-        "TruthfulQA": 0.368,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.154,
-        "XSUM": 0.134,
-        "IMDB": 0.934,
-        "CivilComments": 0.61,
-        "RAFT": 0.699
-      }
-    },
-    {
-      "model_id": "EleutherAI/pythia-12b",
-      "name": "Pythia 12B",
-      "developer": "EleutherAI",
-      "scores": {
-        "Mean win rate": 0.257,
-        "MMLU": 0.274,
-        "BoolQ": 0.662,
-        "NarrativeQA": 0.596,
-        "NaturalQuestions (open-book)": 0.581,
-        "QuAC": 0.313,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.177,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.931,
-        "CivilComments": 0.531,
-        "RAFT": 0.514
-      }
-    },
-    {
-      "model_id": "EleutherAI/pythia-6.9b",
-      "name": "Pythia 6.9B",
-      "developer": "EleutherAI",
-      "scores": {
-        "Mean win rate": 0.196,
-        "MMLU": 0.236,
-        "BoolQ": 0.631,
-        "NarrativeQA": 0.528,
-        "NaturalQuestions (open-book)": 0.539,
-        "QuAC": 0.296,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.213,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.928,
-        "CivilComments": 0.511,
-        "RAFT": 0.502
-      }
-    },
-    {
-      "model_id": "ai21/J1-Grande-v1-17B",
-      "name": "J1-Grande v1 17B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.433,
-        "MMLU": 0.27,
-        "BoolQ": 0.722,
-        "NarrativeQA": 0.672,
-        "NaturalQuestions (open-book)": 0.578,
-        "QuAC": 0.362,
-        "HellaSwag": 0.739,
-        "OpenbookQA": 0.52,
-        "TruthfulQA": 0.193,
-        "MS MARCO (TREC)": 0.341,
-        "CNN/DailyMail": 0.143,
-        "XSUM": 0.122,
-        "IMDB": 0.953,
-        "CivilComments": 0.529,
-        "RAFT": 0.658
-      }
-    },
-    {
-      "model_id": "ai21/J1-Grande-v2-beta-17B",
-      "name": "J1-Grande v2 beta 17B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.706,
-        "MMLU": 0.445,
-        "BoolQ": 0.812,
-        "NarrativeQA": 0.725,
-        "NaturalQuestions (open-book)": 0.625,
-        "QuAC": 0.392,
-        "HellaSwag": 0.764,
-        "OpenbookQA": 0.56,
-        "TruthfulQA": 0.306,
-        "MS MARCO (TREC)": 0.46,
-        "CNN/DailyMail": 0.146,
-        "XSUM": 0.152,
-        "IMDB": 0.957,
-        "CivilComments": 0.546,
-        "RAFT": 0.679
-      }
-    },
-    {
-      "model_id": "ai21/J1-Jumbo-v1-178B",
-      "name": "J1-Jumbo v1 178B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.517,
-        "MMLU": 0.259,
-        "BoolQ": 0.776,
-        "NarrativeQA": 0.695,
-        "NaturalQuestions (open-book)": 0.595,
-        "QuAC": 0.358,
-        "HellaSwag": 0.765,
-        "OpenbookQA": 0.534,
-        "TruthfulQA": 0.175,
-        "MS MARCO (TREC)": 0.363,
-        "CNN/DailyMail": 0.144,
-        "XSUM": 0.129,
-        "IMDB": 0.943,
-        "CivilComments": 0.553,
-        "RAFT": 0.681
-      }
-    },
-    {
-      "model_id": "ai21/J1-Large-v1-7.5B",
-      "name": "J1-Large v1 7.5B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.285,
-        "MMLU": 0.241,
-        "BoolQ": 0.683,
-        "NarrativeQA": 0.623,
-        "NaturalQuestions (open-book)": 0.532,
-        "QuAC": 0.328,
-        "HellaSwag": 0.7,
-        "OpenbookQA": 0.514,
-        "TruthfulQA": 0.197,
-        "MS MARCO (TREC)": 0.292,
-        "CNN/DailyMail": 0.134,
-        "XSUM": 0.102,
-        "IMDB": 0.956,
-        "CivilComments": 0.532,
-        "RAFT": 0.545
-      }
-    },
-    {
-      "model_id": "ai21/Jurassic-2-Grande-17B",
-      "name": "Jurassic-2 Grande 17B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.743,
-        "MMLU": 0.475,
-        "BoolQ": 0.826,
-        "NarrativeQA": 0.737,
-        "NaturalQuestions (open-book)": 0.639,
-        "QuAC": 0.418,
-        "HellaSwag": 0.781,
-        "OpenbookQA": 0.542,
-        "TruthfulQA": 0.348,
-        "MS MARCO (TREC)": 0.514,
-        "CNN/DailyMail": 0.144,
-        "XSUM": 0.167,
-        "IMDB": 0.938,
-        "CivilComments": 0.547,
-        "RAFT": 0.712
-      }
-    },
-    {
-      "model_id": "ai21/Jurassic-2-Jumbo-178B",
-      "name": "Jurassic-2 Jumbo 178B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.824,
-        "MMLU": 0.48,
-        "BoolQ": 0.829,
-        "NarrativeQA": 0.733,
-        "NaturalQuestions (open-book)": 0.669,
-        "QuAC": 0.435,
-        "HellaSwag": 0.788,
-        "OpenbookQA": 0.558,
-        "TruthfulQA": 0.437,
-        "MS MARCO (TREC)": 0.661,
-        "CNN/DailyMail": 0.149,
-        "XSUM": 0.182,
-        "IMDB": 0.938,
-        "CivilComments": 0.57,
-        "RAFT": 0.746
-      }
-    },
-    {
-      "model_id": "ai21/Jurassic-2-Large-7.5B",
-      "name": "Jurassic-2 Large 7.5B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.553,
-        "MMLU": 0.339,
-        "BoolQ": 0.742,
-        "NarrativeQA": -1,
-        "NaturalQuestions (open-book)": 0.589,
-        "QuAC": -1,
-        "HellaSwag": 0.729,
-        "OpenbookQA": 0.53,
-        "TruthfulQA": 0.245,
-        "MS MARCO (TREC)": 0.464,
-        "CNN/DailyMail": 0.136,
-        "XSUM": 0.142,
-        "IMDB": 0.956,
-        "CivilComments": 0.57,
-        "RAFT": 0.622
-      }
-    },
-    {
-      "model_id": "aleph-alpha/Luminous-Base-13B",
-      "name": "Luminous Base 13B",
-      "developer": "aleph-alpha",
-      "scores": {
-        "Mean win rate": 0.315,
-        "MMLU": 0.27,
-        "BoolQ": 0.719,
-        "NarrativeQA": 0.605,
-        "NaturalQuestions (open-book)": 0.568,
-        "QuAC": 0.334,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.182,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.11,
-        "XSUM": 0.105,
-        "IMDB": 0.939,
-        "CivilComments": 0.544,
-        "RAFT": 0.473
-      }
-    },
-    {
-      "model_id": "aleph-alpha/Luminous-Extended-30B",
-      "name": "Luminous Extended 30B",
-      "developer": "aleph-alpha",
-      "scores": {
-        "Mean win rate": 0.485,
-        "MMLU": 0.321,
-        "BoolQ": 0.767,
-        "NarrativeQA": 0.665,
-        "NaturalQuestions (open-book)": 0.609,
-        "QuAC": 0.349,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.221,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.139,
-        "XSUM": 0.124,
-        "IMDB": 0.947,
-        "CivilComments": 0.524,
-        "RAFT": 0.523
-      }
-    },
-    {
-      "model_id": "aleph-alpha/Luminous-Supreme-70B",
-      "name": "Luminous Supreme 70B",
-      "developer": "aleph-alpha",
-      "scores": {
-        "Mean win rate": 0.662,
-        "MMLU": 0.38,
-        "BoolQ": 0.775,
-        "NarrativeQA": 0.711,
-        "NaturalQuestions (open-book)": 0.649,
-        "QuAC": 0.37,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.222,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.15,
-        "XSUM": 0.136,
-        "IMDB": 0.959,
-        "CivilComments": 0.562,
-        "RAFT": 0.653
-      }
-    },
-    {
-      "model_id": "bigscience/BLOOM-176B",
-      "name": "BLOOM 176B",
-      "developer": "bigscience",
-      "scores": {
-        "Mean win rate": 0.446,
-        "MMLU": 0.299,
-        "BoolQ": 0.704,
-        "NarrativeQA": 0.662,
-        "NaturalQuestions (open-book)": 0.621,
-        "QuAC": 0.361,
-        "HellaSwag": 0.744,
-        "OpenbookQA": 0.534,
-        "TruthfulQA": 0.205,
-        "MS MARCO (TREC)": 0.386,
-        "CNN/DailyMail": 0.08,
-        "XSUM": 0.03,
-        "IMDB": 0.945,
-        "CivilComments": 0.62,
-        "RAFT": 0.592
-      }
-    },
-    {
-      "model_id": "bigscience/T0pp-11B",
-      "name": "T0pp 11B",
-      "developer": "bigscience",
-      "scores": {
-        "Mean win rate": 0.197,
-        "MMLU": 0.407,
-        "BoolQ": 0,
-        "NarrativeQA": 0.151,
-        "NaturalQuestions (open-book)": 0.19,
-        "QuAC": 0.121,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.377,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.122,
-        "XSUM": 0.09,
-        "IMDB": 0.207,
-        "CivilComments": 0.234,
-        "RAFT": 0.118
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-Command-beta-52.4B",
-      "name": "Cohere Command beta 52.4B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.874,
-        "MMLU": 0.452,
-        "BoolQ": 0.856,
-        "NarrativeQA": 0.752,
-        "NaturalQuestions (open-book)": 0.76,
-        "QuAC": 0.432,
-        "HellaSwag": 0.811,
-        "OpenbookQA": 0.582,
-        "TruthfulQA": 0.269,
-        "MS MARCO (TREC)": 0.762,
-        "CNN/DailyMail": 0.161,
-        "XSUM": 0.152,
-        "IMDB": 0.96,
-        "CivilComments": 0.601,
-        "RAFT": 0.667
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-Command-beta-6.1B",
-      "name": "Cohere Command beta 6.1B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.675,
-        "MMLU": 0.406,
-        "BoolQ": 0.798,
-        "NarrativeQA": 0.709,
-        "NaturalQuestions (open-book)": 0.717,
-        "QuAC": 0.375,
-        "HellaSwag": 0.752,
-        "OpenbookQA": 0.55,
-        "TruthfulQA": 0.203,
-        "MS MARCO (TREC)": 0.709,
-        "CNN/DailyMail": 0.153,
-        "XSUM": 0.122,
-        "IMDB": 0.961,
-        "CivilComments": 0.54,
-        "RAFT": 0.634
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-large-v20220720-13.1B",
-      "name": "Cohere large v20220720 13.1B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.372,
-        "MMLU": 0.324,
-        "BoolQ": 0.725,
-        "NarrativeQA": 0.625,
-        "NaturalQuestions (open-book)": 0.573,
-        "QuAC": 0.338,
-        "HellaSwag": 0.736,
-        "OpenbookQA": 0.542,
-        "TruthfulQA": 0.181,
-        "MS MARCO (TREC)": 0.33,
-        "CNN/DailyMail": 0.126,
-        "XSUM": 0.108,
-        "IMDB": 0.933,
-        "CivilComments": 0.507,
-        "RAFT": 0.596
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-medium-v20220720-6.1B",
-      "name": "Cohere medium v20220720 6.1B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.23,
-        "MMLU": 0.279,
-        "BoolQ": 0.659,
-        "NarrativeQA": 0.559,
-        "NaturalQuestions (open-book)": 0.504,
-        "QuAC": 0.279,
-        "HellaSwag": 0.706,
-        "OpenbookQA": 0.496,
-        "TruthfulQA": 0.19,
-        "MS MARCO (TREC)": 0.374,
-        "CNN/DailyMail": 0.077,
-        "XSUM": 0.087,
-        "IMDB": 0.935,
-        "CivilComments": 0.504,
-        "RAFT": 0.52
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-medium-v20221108-6.1B",
-      "name": "Cohere medium v20221108 6.1B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.312,
-        "MMLU": 0.254,
-        "BoolQ": 0.7,
-        "NarrativeQA": 0.61,
-        "NaturalQuestions (open-book)": 0.517,
-        "QuAC": 0.314,
-        "HellaSwag": 0.726,
-        "OpenbookQA": 0.538,
-        "TruthfulQA": 0.215,
-        "MS MARCO (TREC)": 0.373,
-        "CNN/DailyMail": 0.121,
-        "XSUM": 0.099,
-        "IMDB": 0.935,
-        "CivilComments": 0.5,
-        "RAFT": 0.591
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-small-v20220720-410M",
-      "name": "Cohere small v20220720 410M",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.109,
-        "MMLU": 0.264,
-        "BoolQ": 0.457,
-        "NarrativeQA": 0.294,
-        "NaturalQuestions (open-book)": 0.309,
-        "QuAC": 0.219,
-        "HellaSwag": 0.483,
-        "OpenbookQA": 0.348,
-        "TruthfulQA": 0.217,
-        "MS MARCO (TREC)": 0.304,
-        "CNN/DailyMail": 0.063,
-        "XSUM": 0.033,
-        "IMDB": 0.578,
-        "CivilComments": 0.501,
-        "RAFT": 0.492
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-xlarge-v20220609-52.4B",
-      "name": "Cohere xlarge v20220609 52.4B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.56,
-        "MMLU": 0.353,
-        "BoolQ": 0.718,
-        "NarrativeQA": 0.65,
-        "NaturalQuestions (open-book)": 0.595,
-        "QuAC": 0.361,
-        "HellaSwag": 0.811,
-        "OpenbookQA": 0.55,
-        "TruthfulQA": 0.198,
-        "MS MARCO (TREC)": 0.459,
-        "CNN/DailyMail": 0.144,
-        "XSUM": 0.129,
-        "IMDB": 0.956,
-        "CivilComments": 0.532,
-        "RAFT": 0.633
-      }
-    },
-    {
-      "model_id": "cohere/Cohere-xlarge-v20221108-52.4B",
-      "name": "Cohere xlarge v20221108 52.4B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.664,
-        "MMLU": 0.382,
-        "BoolQ": 0.762,
-        "NarrativeQA": 0.672,
-        "NaturalQuestions (open-book)": 0.628,
-        "QuAC": 0.374,
-        "HellaSwag": 0.81,
-        "OpenbookQA": 0.588,
-        "TruthfulQA": 0.169,
-        "MS MARCO (TREC)": 0.55,
-        "CNN/DailyMail": 0.153,
-        "XSUM": 0.153,
-        "IMDB": 0.956,
-        "CivilComments": 0.524,
-        "RAFT": 0.624
-      }
-    },
-    {
-      "model_id": "google/Palmyra-X-43B",
-      "name": "Palmyra X 43B",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.732,
-        "MMLU": 0.609,
-        "BoolQ": 0.896,
-        "NarrativeQA": 0.742,
-        "NaturalQuestions (open-book)": -1,
-        "QuAC": 0.473,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.616,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.049,
-        "XSUM": 0.149,
-        "IMDB": 0.935,
-        "CivilComments": 0.008,
-        "RAFT": 0.701
-      }
-    },
-    {
-      "model_id": "google/T5-11B",
-      "name": "T5 11B",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.131,
-        "MMLU": 0.29,
-        "BoolQ": 0.761,
-        "NarrativeQA": 0.086,
-        "NaturalQuestions (open-book)": 0.477,
-        "QuAC": 0.116,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.133,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.043,
-        "XSUM": 0.015,
-        "IMDB": 0.379,
-        "CivilComments": 0.509,
-        "RAFT": 0.37
-      }
-    },
-    {
-      "model_id": "google/UL2-20B",
-      "name": "UL2 20B",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.167,
-        "MMLU": 0.291,
-        "BoolQ": 0.746,
-        "NarrativeQA": 0.083,
-        "NaturalQuestions (open-book)": 0.349,
-        "QuAC": 0.144,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.193,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.03,
-        "XSUM": 0.058,
-        "IMDB": 0.337,
-        "CivilComments": 0.521,
-        "RAFT": 0.404
-      }
-    },
-    {
-      "model_id": "lmsys/Vicuna-v1.3-13B",
-      "name": "Vicuna v1.3 13B",
-      "developer": "lmsys",
-      "scores": {
-        "Mean win rate": 0.706,
-        "MMLU": 0.462,
-        "BoolQ": 0.808,
-        "NarrativeQA": 0.691,
-        "NaturalQuestions (open-book)": 0.686,
-        "QuAC": 0.403,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.385,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.762,
-        "CivilComments": 0.645,
-        "RAFT": 0.657
-      }
-    },
-    {
-      "model_id": "lmsys/Vicuna-v1.3-7B",
-      "name": "Vicuna v1.3 7B",
-      "developer": "lmsys",
-      "scores": {
-        "Mean win rate": 0.625,
-        "MMLU": 0.434,
-        "BoolQ": 0.76,
-        "NarrativeQA": 0.643,
-        "NaturalQuestions (open-book)": 0.634,
-        "QuAC": 0.392,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.292,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.916,
-        "CivilComments": 0.62,
-        "RAFT": 0.693
-      }
-    },
-    {
-      "model_id": "meta/LLaMA-13B",
-      "name": "LLaMA 13B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.595,
-        "MMLU": 0.422,
-        "BoolQ": 0.714,
-        "NarrativeQA": 0.711,
-        "NaturalQuestions (open-book)": 0.614,
-        "QuAC": 0.347,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.324,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.928,
-        "CivilComments": 0.6,
-        "RAFT": 0.643
-      }
-    },
-    {
-      "model_id": "meta/LLaMA-30B",
-      "name": "LLaMA 30B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.781,
-        "MMLU": 0.531,
-        "BoolQ": 0.861,
-        "NarrativeQA": 0.752,
-        "NaturalQuestions (open-book)": 0.666,
-        "QuAC": 0.39,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.344,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.927,
-        "CivilComments": 0.549,
-        "RAFT": 0.752
-      }
-    },
-    {
-      "model_id": "meta/LLaMA-65B",
-      "name": "LLaMA 65B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.908,
-        "MMLU": 0.584,
-        "BoolQ": 0.871,
-        "NarrativeQA": 0.755,
-        "NaturalQuestions (open-book)": 0.672,
-        "QuAC": 0.401,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.508,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.962,
-        "CivilComments": 0.655,
-        "RAFT": 0.702
-      }
-    },
-    {
-      "model_id": "meta/LLaMA-7B",
-      "name": "LLaMA 7B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.533,
-        "MMLU": 0.321,
-        "BoolQ": 0.756,
-        "NarrativeQA": 0.669,
-        "NaturalQuestions (open-book)": 0.589,
-        "QuAC": 0.338,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.28,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.947,
-        "CivilComments": 0.563,
-        "RAFT": 0.573
-      }
-    },
-    {
-      "model_id": "meta/OPT-175B",
-      "name": "OPT 175B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.609,
-        "MMLU": 0.318,
-        "BoolQ": 0.793,
-        "NarrativeQA": 0.671,
-        "NaturalQuestions (open-book)": 0.615,
-        "QuAC": 0.36,
-        "HellaSwag": 0.791,
-        "OpenbookQA": 0.586,
-        "TruthfulQA": 0.25,
-        "MS MARCO (TREC)": 0.448,
-        "CNN/DailyMail": 0.146,
-        "XSUM": 0.155,
-        "IMDB": 0.947,
-        "CivilComments": 0.505,
-        "RAFT": 0.606
-      }
-    },
-    {
-      "model_id": "meta/OPT-66B",
-      "name": "OPT 66B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.448,
-        "MMLU": 0.276,
-        "BoolQ": 0.76,
-        "NarrativeQA": 0.638,
-        "NaturalQuestions (open-book)": 0.596,
-        "QuAC": 0.357,
-        "HellaSwag": 0.745,
-        "OpenbookQA": 0.534,
-        "TruthfulQA": 0.201,
-        "MS MARCO (TREC)": 0.482,
-        "CNN/DailyMail": 0.136,
-        "XSUM": 0.126,
-        "IMDB": 0.917,
-        "CivilComments": 0.506,
-        "RAFT": 0.557
-      }
-    },
-    {
-      "model_id": "meta/llama-2-13b",
-      "name": "Llama 2 13B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.823,
-        "MMLU": 0.507,
-        "BoolQ": 0.811,
-        "NarrativeQA": 0.744,
-        "NaturalQuestions (open-book)": 0.637,
-        "QuAC": 0.424,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.33,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.962,
-        "CivilComments": 0.588,
-        "RAFT": 0.707
-      }
-    },
-    {
-      "model_id": "meta/llama-2-70b",
-      "name": "Llama 2 70B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.944,
-        "MMLU": 0.582,
-        "BoolQ": 0.886,
-        "NarrativeQA": 0.77,
-        "NaturalQuestions (open-book)": 0.674,
-        "QuAC": 0.484,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.554,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.961,
-        "CivilComments": 0.652,
-        "RAFT": 0.727
-      }
-    },
-    {
-      "model_id": "meta/llama-2-7b",
-      "name": "Llama 2 7B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.607,
-        "MMLU": 0.431,
-        "BoolQ": 0.762,
-        "NarrativeQA": 0.691,
-        "NaturalQuestions (open-book)": 0.611,
-        "QuAC": 0.406,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.272,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.907,
-        "CivilComments": 0.562,
-        "RAFT": 0.643
-      }
-    },
-    {
-      "model_id": "microsoft/TNLG-v2-530B",
-      "name": "TNLG v2 530B",
-      "developer": "microsoft",
-      "scores": {
-        "Mean win rate": 0.787,
-        "MMLU": 0.469,
-        "BoolQ": 0.809,
-        "NarrativeQA": 0.722,
-        "NaturalQuestions (open-book)": 0.642,
-        "QuAC": 0.39,
-        "HellaSwag": 0.799,
-        "OpenbookQA": 0.562,
-        "TruthfulQA": 0.251,
-        "MS MARCO (TREC)": 0.643,
-        "CNN/DailyMail": 0.161,
-        "XSUM": 0.169,
-        "IMDB": 0.941,
-        "CivilComments": 0.601,
-        "RAFT": 0.679
-      }
-    },
-    {
-      "model_id": "microsoft/TNLG-v2-6.7B",
-      "name": "TNLG v2 6.7B",
-      "developer": "microsoft",
-      "scores": {
-        "Mean win rate": 0.309,
-        "MMLU": 0.242,
-        "BoolQ": 0.698,
-        "NarrativeQA": 0.631,
-        "NaturalQuestions (open-book)": 0.561,
-        "QuAC": 0.345,
-        "HellaSwag": 0.704,
-        "OpenbookQA": 0.478,
-        "TruthfulQA": 0.167,
-        "MS MARCO (TREC)": 0.332,
-        "CNN/DailyMail": 0.146,
-        "XSUM": 0.11,
-        "IMDB": 0.927,
-        "CivilComments": 0.532,
-        "RAFT": 0.525
-      }
-    },
-    {
-      "model_id": "mistralai/Mistral-v0.1-7B",
-      "name": "Mistral v0.1 7B",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.884,
-        "MMLU": 0.572,
-        "BoolQ": 0.874,
-        "NarrativeQA": 0.716,
-        "NaturalQuestions (open-book)": 0.687,
-        "QuAC": 0.423,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.422,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.962,
-        "CivilComments": 0.624,
-        "RAFT": 0.707
-      }
-    },
-    {
-      "model_id": "mosaicml/MPT-30B",
-      "name": "MPT 30B",
-      "developer": "mosaicml",
-      "scores": {
-        "Mean win rate": 0.714,
-        "MMLU": 0.437,
-        "BoolQ": 0.704,
-        "NarrativeQA": 0.732,
-        "NaturalQuestions (open-book)": 0.673,
-        "QuAC": 0.393,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.231,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.959,
-        "CivilComments": 0.599,
-        "RAFT": 0.723
-      }
-    },
-    {
-      "model_id": "mosaicml/MPT-Instruct-30B",
-      "name": "MPT-Instruct 30B",
-      "developer": "mosaicml",
-      "scores": {
-        "Mean win rate": 0.716,
-        "MMLU": 0.444,
-        "BoolQ": 0.85,
-        "NarrativeQA": 0.733,
-        "NaturalQuestions (open-book)": 0.697,
-        "QuAC": 0.327,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.234,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.956,
-        "CivilComments": 0.573,
-        "RAFT": 0.68
-      }
-    },
-    {
-      "model_id": "openai/GPT-J-6B",
-      "name": "GPT-J 6B",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.273,
-        "MMLU": 0.249,
-        "BoolQ": 0.649,
-        "NarrativeQA": 0.545,
-        "NaturalQuestions (open-book)": 0.559,
-        "QuAC": 0.33,
-        "HellaSwag": 0.663,
-        "OpenbookQA": 0.514,
-        "TruthfulQA": 0.199,
-        "MS MARCO (TREC)": 0.345,
-        "CNN/DailyMail": 0.131,
-        "XSUM": 0.096,
-        "IMDB": 0.939,
-        "CivilComments": 0.52,
-        "RAFT": 0.619
-      }
-    },
-    {
-      "model_id": "openai/GPT-NeoX-20B",
-      "name": "GPT-NeoX 20B",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.351,
-        "MMLU": 0.276,
-        "BoolQ": 0.683,
-        "NarrativeQA": 0.599,
-        "NaturalQuestions (open-book)": 0.596,
-        "QuAC": 0.326,
-        "HellaSwag": 0.718,
-        "OpenbookQA": 0.524,
-        "TruthfulQA": 0.216,
-        "MS MARCO (TREC)": 0.398,
-        "CNN/DailyMail": 0.123,
-        "XSUM": 0.102,
-        "IMDB": 0.948,
-        "CivilComments": 0.516,
-        "RAFT": 0.505
-      }
-    },
-    {
-      "model_id": "openai/ada-350M",
-      "name": "ada 350M",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.108,
-        "MMLU": 0.243,
-        "BoolQ": 0.581,
-        "NarrativeQA": 0.326,
-        "NaturalQuestions (open-book)": 0.365,
-        "QuAC": 0.242,
-        "HellaSwag": 0.435,
-        "OpenbookQA": 0.38,
-        "TruthfulQA": 0.215,
-        "MS MARCO (TREC)": 0.29,
-        "CNN/DailyMail": 0.09,
-        "XSUM": 0.022,
-        "IMDB": 0.849,
-        "CivilComments": 0.517,
-        "RAFT": 0.423
-      }
-    },
-    {
-      "model_id": "openai/babbage-1.3B",
-      "name": "babbage 1.3B",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.114,
-        "MMLU": 0.235,
-        "BoolQ": 0.574,
-        "NarrativeQA": 0.491,
-        "NaturalQuestions (open-book)": 0.451,
-        "QuAC": 0.273,
-        "HellaSwag": 0.555,
-        "OpenbookQA": 0.438,
-        "TruthfulQA": 0.188,
-        "MS MARCO (TREC)": 0.317,
-        "CNN/DailyMail": 0.079,
-        "XSUM": 0.045,
-        "IMDB": 0.597,
-        "CivilComments": 0.519,
-        "RAFT": 0.455
-      }
-    },
-    {
-      "model_id": "openai/curie-6.7B",
-      "name": "curie 6.7B",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.247,
-        "MMLU": 0.243,
-        "BoolQ": 0.656,
-        "NarrativeQA": 0.604,
-        "NaturalQuestions (open-book)": 0.552,
-        "QuAC": 0.321,
-        "HellaSwag": 0.682,
-        "OpenbookQA": 0.502,
-        "TruthfulQA": 0.232,
-        "MS MARCO (TREC)": 0.3,
-        "CNN/DailyMail": 0.113,
-        "XSUM": 0.091,
-        "IMDB": 0.889,
-        "CivilComments": 0.539,
-        "RAFT": 0.49
-      }
-    },
-    {
-      "model_id": "openai/davinci-175B",
-      "name": "davinci 175B",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.538,
-        "MMLU": 0.422,
-        "BoolQ": 0.722,
-        "NarrativeQA": 0.687,
-        "NaturalQuestions (open-book)": 0.625,
-        "QuAC": 0.36,
-        "HellaSwag": 0.775,
-        "OpenbookQA": 0.586,
-        "TruthfulQA": 0.194,
-        "MS MARCO (TREC)": 0.378,
-        "CNN/DailyMail": 0.127,
-        "XSUM": 0.126,
-        "IMDB": 0.933,
-        "CivilComments": 0.532,
-        "RAFT": 0.642
-      }
-    },
-    {
-      "model_id": "openai/gpt-3.5-turbo-0301",
-      "name": "gpt-3.5-turbo-0301",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.76,
-        "MMLU": 0.59,
-        "BoolQ": 0.74,
-        "NarrativeQA": 0.663,
-        "NaturalQuestions (open-book)": 0.624,
-        "QuAC": 0.512,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.609,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.899,
-        "CivilComments": 0.674,
-        "RAFT": 0.768
-      }
-    },
-    {
-      "model_id": "openai/gpt-3.5-turbo-0613",
-      "name": "GPT-3.5 Turbo 0613",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.783,
-        "MMLU": 0.391,
-        "BoolQ": 0.87,
-        "NarrativeQA": 0.625,
-        "NaturalQuestions (open-book)": 0.675,
-        "QuAC": 0.485,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.339,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.943,
-        "CivilComments": 0.696,
-        "RAFT": 0.748
-      }
-    },
-    {
-      "model_id": "openai/text-ada-001",
-      "name": "text-ada-001",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.107,
-        "MMLU": 0.238,
-        "BoolQ": 0.464,
-        "NarrativeQA": 0.238,
-        "NaturalQuestions (open-book)": 0.149,
-        "QuAC": 0.176,
-        "HellaSwag": 0.429,
-        "OpenbookQA": 0.346,
-        "TruthfulQA": 0.232,
-        "MS MARCO (TREC)": 0.302,
-        "CNN/DailyMail": 0.136,
-        "XSUM": 0.034,
-        "IMDB": 0.822,
-        "CivilComments": 0.503,
-        "RAFT": 0.406
-      }
-    },
-    {
-      "model_id": "openai/text-babbage-001",
-      "name": "text-babbage-001",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.229,
-        "MMLU": 0.229,
-        "BoolQ": 0.451,
-        "NarrativeQA": 0.429,
-        "NaturalQuestions (open-book)": 0.33,
-        "QuAC": 0.284,
-        "HellaSwag": 0.561,
-        "OpenbookQA": 0.452,
-        "TruthfulQA": 0.233,
-        "MS MARCO (TREC)": 0.449,
-        "CNN/DailyMail": 0.151,
-        "XSUM": 0.046,
-        "IMDB": 0.913,
-        "CivilComments": 0.499,
-        "RAFT": 0.509
-      }
-    },
-    {
-      "model_id": "openai/text-curie-001",
-      "name": "text-curie-001",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.36,
-        "MMLU": 0.237,
-        "BoolQ": 0.62,
-        "NarrativeQA": 0.582,
-        "NaturalQuestions (open-book)": 0.571,
-        "QuAC": 0.358,
-        "HellaSwag": 0.676,
-        "OpenbookQA": 0.514,
-        "TruthfulQA": 0.257,
-        "MS MARCO (TREC)": 0.507,
-        "CNN/DailyMail": 0.152,
-        "XSUM": 0.076,
-        "IMDB": 0.923,
-        "CivilComments": 0.537,
-        "RAFT": 0.489
-      }
-    },
-    {
-      "model_id": "openai/text-davinci-002",
-      "name": "GPT-3.5 text-davinci-002",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.905,
-        "MMLU": 0.568,
-        "BoolQ": 0.877,
-        "NarrativeQA": 0.727,
-        "NaturalQuestions (open-book)": 0.713,
-        "QuAC": 0.445,
-        "HellaSwag": 0.815,
-        "OpenbookQA": 0.594,
-        "TruthfulQA": 0.61,
-        "MS MARCO (TREC)": 0.664,
-        "CNN/DailyMail": 0.153,
-        "XSUM": 0.144,
-        "IMDB": 0.948,
-        "CivilComments": 0.668,
-        "RAFT": 0.733
-      }
-    },
-    {
-      "model_id": "openai/text-davinci-003",
-      "name": "GPT-3.5 text-davinci-003",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.872,
-        "MMLU": 0.569,
-        "BoolQ": 0.881,
-        "NarrativeQA": 0.727,
-        "NaturalQuestions (open-book)": 0.77,
-        "QuAC": 0.525,
-        "HellaSwag": 0.822,
-        "OpenbookQA": 0.646,
-        "TruthfulQA": 0.593,
-        "MS MARCO (TREC)": 0.644,
-        "CNN/DailyMail": 0.156,
-        "XSUM": 0.124,
-        "IMDB": 0.848,
-        "CivilComments": 0.684,
-        "RAFT": 0.759
-      }
-    },
-    {
-      "model_id": "stanford/Alpaca-7B",
-      "name": "Alpaca 7B",
-      "developer": "stanford",
-      "scores": {
-        "Mean win rate": 0.381,
-        "MMLU": 0.385,
-        "BoolQ": 0.778,
-        "NarrativeQA": 0.396,
-        "NaturalQuestions (open-book)": 0.592,
-        "QuAC": 0.27,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.243,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.738,
-        "CivilComments": 0.566,
-        "RAFT": 0.486
-      }
-    },
-    {
-      "model_id": "tiiuae/Falcon-Instruct-40B",
-      "name": "Falcon-Instruct 40B",
-      "developer": "tiiuae",
-      "scores": {
-        "Mean win rate": 0.727,
-        "MMLU": 0.497,
-        "BoolQ": 0.829,
-        "NarrativeQA": 0.625,
-        "NaturalQuestions (open-book)": 0.666,
-        "QuAC": 0.371,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.384,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.959,
-        "CivilComments": 0.603,
-        "RAFT": 0.586
-      }
-    },
-    {
-      "model_id": "tiiuae/Falcon-Instruct-7B",
-      "name": "Falcon-Instruct 7B",
-      "developer": "tiiuae",
-      "scores": {
-        "Mean win rate": 0.244,
-        "MMLU": 0.275,
-        "BoolQ": 0.72,
-        "NarrativeQA": 0.476,
-        "NaturalQuestions (open-book)": 0.449,
-        "QuAC": 0.311,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.213,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.852,
-        "CivilComments": 0.511,
-        "RAFT": 0.523
-      }
-    },
-    {
-      "model_id": "tiiuae/falcon-40b",
-      "name": "Falcon 40B",
-      "developer": "tiiuae",
-      "scores": {
-        "Mean win rate": 0.729,
-        "MMLU": 0.509,
-        "BoolQ": 0.819,
-        "NarrativeQA": 0.673,
-        "NaturalQuestions (open-book)": 0.675,
-        "QuAC": 0.307,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.353,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.959,
-        "CivilComments": 0.552,
-        "RAFT": 0.661
-      }
-    },
-    {
-      "model_id": "tiiuae/falcon-7b",
-      "name": "Falcon 7B",
-      "developer": "tiiuae",
-      "scores": {
-        "Mean win rate": 0.378,
-        "MMLU": 0.286,
-        "BoolQ": 0.753,
-        "NarrativeQA": 0.621,
-        "NaturalQuestions (open-book)": 0.579,
-        "QuAC": 0.332,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.234,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.836,
-        "CivilComments": 0.514,
-        "RAFT": 0.602
-      }
-    },
-    {
-      "model_id": "together/RedPajama-INCITE-Base-7B",
-      "name": "RedPajama-INCITE-Base 7B",
-      "developer": "together",
-      "scores": {
-        "Mean win rate": 0.378,
-        "MMLU": 0.302,
-        "BoolQ": 0.713,
-        "NarrativeQA": 0.617,
-        "NaturalQuestions (open-book)": 0.586,
-        "QuAC": 0.336,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.205,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.752,
-        "CivilComments": 0.547,
-        "RAFT": 0.648
-      }
-    },
-    {
-      "model_id": "together/RedPajama-INCITE-Base-v1-3B",
-      "name": "RedPajama-INCITE-Base-v1 3B",
-      "developer": "together",
-      "scores": {
-        "Mean win rate": 0.311,
-        "MMLU": 0.263,
-        "BoolQ": 0.685,
-        "NarrativeQA": 0.555,
-        "NaturalQuestions (open-book)": 0.52,
-        "QuAC": 0.309,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.277,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.907,
-        "CivilComments": 0.549,
-        "RAFT": 0.502
-      }
-    },
-    {
-      "model_id": "together/RedPajama-INCITE-Instruct-7B",
-      "name": "RedPajama-INCITE-Instruct 7B",
-      "developer": "together",
-      "scores": {
-        "Mean win rate": 0.524,
-        "MMLU": 0.363,
-        "BoolQ": 0.705,
-        "NarrativeQA": 0.638,
-        "NaturalQuestions (open-book)": 0.659,
-        "QuAC": 0.26,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.243,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.927,
-        "CivilComments": 0.664,
-        "RAFT": 0.695
-      }
-    },
-    {
-      "model_id": "together/RedPajama-INCITE-Instruct-v1-3B",
-      "name": "RedPajama-INCITE-Instruct-v1 3B",
-      "developer": "together",
-      "scores": {
-        "Mean win rate": 0.366,
-        "MMLU": 0.257,
-        "BoolQ": 0.677,
-        "NarrativeQA": 0.638,
-        "NaturalQuestions (open-book)": 0.637,
-        "QuAC": 0.259,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.208,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": -1,
-        "XSUM": -1,
-        "IMDB": 0.894,
-        "CivilComments": 0.549,
-        "RAFT": 0.661
-      }
-    },
-    {
-      "model_id": "writer/InstructPalmyra-30B",
-      "name": "InstructPalmyra 30B",
-      "developer": "writer",
-      "scores": {
-        "Mean win rate": 0.568,
-        "MMLU": 0.403,
-        "BoolQ": 0.751,
-        "NarrativeQA": 0.496,
-        "NaturalQuestions (open-book)": 0.682,
-        "QuAC": 0.433,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.185,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.152,
-        "XSUM": 0.104,
-        "IMDB": 0.94,
-        "CivilComments": 0.555,
-        "RAFT": 0.652
-      }
-    },
-    {
-      "model_id": "yandex/YaLM-100B",
-      "name": "YaLM 100B",
-      "developer": "yandex",
-      "scores": {
-        "Mean win rate": 0.075,
-        "MMLU": 0.243,
-        "BoolQ": 0.634,
-        "NarrativeQA": 0.252,
-        "NaturalQuestions (open-book)": 0.227,
-        "QuAC": 0.162,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.202,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.017,
-        "XSUM": 0.021,
-        "IMDB": 0.836,
-        "CivilComments": 0.49,
-        "RAFT": 0.395
-      }
-    },
-    {
-      "model_id": "zhipu-ai/GLM-130B",
-      "name": "GLM 130B",
-      "developer": "zhipu-ai",
-      "scores": {
-        "Mean win rate": 0.512,
-        "MMLU": 0.344,
-        "BoolQ": 0.784,
-        "NarrativeQA": 0.706,
-        "NaturalQuestions (open-book)": 0.642,
-        "QuAC": 0.272,
-        "HellaSwag": -1,
-        "OpenbookQA": -1,
-        "TruthfulQA": 0.218,
-        "MS MARCO (TREC)": -1,
-        "CNN/DailyMail": 0.154,
-        "XSUM": 0.132,
-        "IMDB": 0.955,
-        "CivilComments": 0.5,
-        "RAFT": 0.598
-      }
-    }
-  ]
-}

data/benchmarks/helm_instruct.json

@@ -1,60 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/claude-v1.3",
-      "name": "Anthropic Claude v1.3",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.611,
-        "Anthropic RLHF dataset": 4.965,
-        "Best ChatGPT Prompts": 4.995,
-        "Koala test dataset": 4.981,
-        "Open Assistant": 4.975,
-        "Self Instruct": 4.992,
-        "Vicuna": 4.989
-      }
-    },
-    {
-      "model_id": "cohere/command-xlarge-beta",
-      "name": "Cohere Command beta 52.4B",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.089,
-        "Anthropic RLHF dataset": 4.214,
-        "Best ChatGPT Prompts": 4.988,
-        "Koala test dataset": 4.969,
-        "Open Assistant": 4.967,
-        "Self Instruct": 4.971,
-        "Vicuna": 4.995
-      }
-    },
-    {
-      "model_id": "openai/gpt-3.5-turbo-0613",
-      "name": "GPT-3.5 Turbo 0613",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.689,
-        "Anthropic RLHF dataset": 4.964,
-        "Best ChatGPT Prompts": 4.986,
-        "Koala test dataset": 4.987,
-        "Open Assistant": 4.987,
-        "Self Instruct": 4.99,
-        "Vicuna": 4.992
-      }
-    },
-    {
-      "model_id": "openai/gpt-4-0314",
-      "name": "GPT-4 0314",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.611,
-        "Anthropic RLHF dataset": 4.934,
-        "Best ChatGPT Prompts": 4.973,
-        "Koala test dataset": 4.966,
-        "Open Assistant": 4.986,
-        "Self Instruct": 4.976,
-        "Vicuna": 4.995
-      }
-    }
-  ]
-}

data/benchmarks/helm_lite.json

@@ -1,1893 +0,0 @@
-{
-  "benchmark_cards": {
-    "GSM8K": {
-      "benchmark_details": {
-        "name": "GSM8K",
-        "overview": "GSM8K is a benchmark that measures the ability of language models to perform multi-step mathematical reasoning. It consists of 8.5K high-quality, linguistically diverse grade school math word problems. The problems are distinctive because they require 2 to 8 steps to solve using basic arithmetic, and even the largest transformer models struggle to achieve high test performance on them. Solutions are provided in natural language with step-by-step reasoning.",
-        "data_type": "text",
-        "domains": [
-          "grade school mathematics",
-          "math word problems"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "Not specified"
-        ],
-        "resources": [
-          "https://arxiv.org/abs/2110.14168",
-          "https://huggingface.co/datasets/openai/gsm8k",
-          "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To diagnose the failures of current language models in robust multi-step mathematical reasoning and to support research, particularly in methods like training verifiers to judge solution correctness. It also aims to shed light on the properties of large language models' reasoning processes.",
-        "audience": [
-          "Researchers working on language model capabilities and mathematical reasoning"
-        ],
-        "tasks": [
-          "Solving grade school math word problems",
-          "Text generation for question answering"
-        ],
-        "limitations": "Even the largest models struggle with high test performance on this dataset, and autoregressive models have no mechanism to correct their own errors during solution generation.",
-        "out_of_scope_uses": [
-          "Not specified"
-        ]
-      },
-      "data": {
-        "source": "The dataset was created by hiring freelance contractors via Upwork and then scaled using the NLP data labeling platform Surge AI. Problems and solutions were written by these contractors.",
-        "size": "8.5K (8,500) problems, with a size category of 10K<n<100K. The training set contains 7,473 examples and the test set contains 1,319 examples.",
-        "format": "parquet. The data is structured with a 'Problem:' field followed by a 'Solution:' field, where the solution includes step-by-step reasoning with intermediate calculations in special tags (e.g., `<<4*2=8>>`) and ends with a 'Final Answer:'.",
-        "annotation": "Contractors wrote the problems and solutions. For verification, different workers re-solved all problems to check agreement with the original solutions; problematic problems were either repaired or discarded. The annotators were from Surge AI."
-      },
-      "methodology": {
-        "methods": [
-          "Models are evaluated by generating step-by-step solutions to math word problems. The dataset provides two answer formats: a standard step-by-step solution and a solution structured with Socratic sub-questions.",
-          "The paper proposes a verification method where a separate verifier model is trained to judge the correctness of generated solutions. At test time, multiple candidate solutions are generated, and the one ranked highest by the verifier is selected."
-        ],
-        "metrics": [
-          "GSM8K"
-        ],
-        "calculation": "The GSM8K metric is a continuous score where higher values are better. It is described as 'EM on GSM8K', indicating it measures exact match accuracy.",
-        "interpretation": "Higher scores indicate better performance. The score is not bounded, but typical model performance ranges from low to high, with the highest reported score being 75.2.",
-        "baseline_results": "Paper baselines: The paper notes that even the largest transformer models fail to achieve high test performance, but does not report specific scores. EEE results: Llama 3.1 8B Instruct scored 75.2, and Yi 34B scored 0.648 on GSM8K.",
-        "validation": "The paper provides empirical evidence that the verification method scales more effectively with data than a finetuning baseline and remains effective even with a verifier much smaller than the generator."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "Not specified",
-        "data_licensing": "MIT License",
-        "consent_procedures": "Not specified",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Reproducibility",
-          "description": [
-            "Replicating agent behavior or output can be impacted by changes or updates made to external services and tools. This impact is increased if the agent is built with generative AI."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/reproducibility-agentic.html"
-        },
-        {
-          "category": "Incomplete advice",
-          "description": [
-            "When a model provides advice without having enough information, resulting in possible harm if the advice is followed."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/incomplete-advice.html"
-        },
-        {
-          "category": "Improper usage",
-          "description": [
-            "Improper usage occurs when a model is used for a purpose that it was not originally designed for."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "benchmark_details.similar_benchmarks",
-        "purpose_and_intended_users.out_of_scope_uses",
-        "ethical_and_legal_considerations.privacy_and_anonymity",
-        "ethical_and_legal_considerations.consent_procedures",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T15:37:16.459776",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    },
-    "LegalBench": {
-      "benchmark_details": {
-        "name": "LEGALBENCH",
-        "overview": "LEGALBENCH is a benchmark designed to measure the legal reasoning capabilities of large language models. It comprises 162 tasks collaboratively constructed and hand-crafted by legal professionals, covering six distinct types of legal reasoning.",
-        "data_type": "text",
-        "domains": [
-          "legal",
-          "law",
-          "finance"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "GLUE",
-          "HELM",
-          "BigBench",
-          "RAFT"
-        ],
-        "resources": [
-          "https://arxiv.org/abs/2308.11462",
-          "https://huggingface.co/datasets/nguha/legalbench",
-          "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To enable greater study of what types of legal reasoning large language models (LLMs) can perform.",
-        "audience": [
-          "Practitioners (to integrate LLMs into workflows)",
-          "Legal academics",
-          "Computer scientists"
-        ],
-        "tasks": [
-          "Text classification",
-          "Question answering",
-          "Text generation",
-          "Rule-application tasks"
-        ],
-        "limitations": "The tasks do not generalize to all legal reasoning tasks or all types of legal documents, offering only a preliminary understanding of LLM performance.",
-        "out_of_scope_uses": [
-          "Predicting the legality of real-world events",
-          "Predicting the outcome of lawsuits",
-          "Providing legal advice"
-        ]
-      },
-      "data": {
-        "source": "Data is drawn from three categories: existing publicly available datasets and corpora (some reformatted), datasets previously created by legal professionals but not released, and tasks developed specifically for LegalBench. The tasks originate from 36 distinct corpora.",
-        "size": "The benchmark comprises 162 tasks. The distribution of tasks by sample count is: 28 tasks have 50-100 samples, 97 have 100-500 samples, 29 have 500-2000 samples, and 8 have 2000+ samples. The overall dataset falls into the size category of 10,000 to 100,000 examples.",
-        "format": "Examples are presented in a structured format with fields such as 'Task name', 'Question', 'Options', and 'Answer'.",
-        "annotation": "Annotation procedures are task-dependent. For certain tasks, each data point was manually validated by a law-trained expert. Detailed annotation methodology for each task is documented in a separate section of the paper."
-      },
-      "methodology": {
-        "methods": [
-          "Models are evaluated in a few-shot setting. Train splits consist of a small random sample of between 2 to 8 instances to capture a true few-shot learning scenario.",
-          "For rule-application tasks, a law-trained expert manually validates each model generation."
-        ],
-        "metrics": [
-          "LegalBench",
-          "Correctness",
-          "Analysis"
-        ],
-        "calculation": "The primary benchmark metric is Exact Match (EM) on LegalBench. For rule-application tasks, two separate metrics are computed: 'correctness' (the proportion of generations without errors) and 'analysis'.",
-        "interpretation": "Higher scores on the LegalBench metric indicate better performance. The metric is continuous and lower scores are not better.",
-        "baseline_results": "The original paper evaluated 20 LLMs from 11 different families but did not provide specific scores. In a separate evaluation suite, the Yi 34B model achieved a score of 0.618.",
-        "validation": "For rule-application tasks, a law-trained expert manually validated each model generation. For datasets reused or adapted from other sources, the original data sheets document any redactions or missing data."
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "For tasks that reuse or adapt existing datasets, the benchmark refers to the original data sheets for details on any data redactions or missing information.",
-        "data_licensing": "other",
-        "consent_procedures": "Not specified.",
-        "compliance_with_regulations": "The benchmark includes a section for each task intended to provide information relevant to ethical review processes, but specific details are not provided in the available facts."
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        },
-        {
-          "category": "Improper usage",
-          "description": [
-            "Improper usage occurs when a model is used for a purpose that it was not originally designed for."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/improper-usage.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "ethical_and_legal_considerations.consent_procedures"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T12:59:10.203815",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    },
-    "MedQA": {
-      "benchmark_details": {
-        "name": "MEDQA",
-        "overview": "MEDQA is a free-form multiple-choice open-domain question answering (OpenQA) benchmark designed to measure a model's ability to solve medical problems. It is distinctive as the first such dataset sourced from professional medical board exams, covering multiple languages and presenting a challenging real-world scenario.",
-        "data_type": "text",
-        "domains": [
-          "medical knowledge",
-          "professional medical exams"
-        ],
-        "languages": [
-          "English"
-        ],
-        "similar_benchmarks": [
-          "ARC",
-          "OpenBookQA"
-        ],
-        "resources": [
-          "https://github.com/jind11/MedQA",
-          "https://arxiv.org/abs/2009.13081",
-          "https://huggingface.co/datasets/GBaker/MedQA-USMLE-4-options",
-          "https://storage.googleapis.com/crfm-helm-public/lite/benchmark_output/releases/v1.13.0/groups/core_scenarios.json"
-        ]
-      },
-      "purpose_and_intended_users": {
-        "goal": "To present a challenging open-domain question answering dataset to promote the development of stronger models capable of handling sophisticated real-world medical scenarios, specifically evaluating performance on medical knowledge as tested in professional exams.",
-        "audience": [
-          "The natural language processing (NLP) community"
-        ],
-        "tasks": [
-          "Free-form multiple-choice question answering",
-          "Open-domain question answering"
-        ],
-        "limitations": "Even the best current methods achieve relatively low accuracy (36.7% to 70.1% across languages), indicating the benchmark's difficulty and the limitations of existing models.",
-        "out_of_scope_uses": [
-          "Not specified"
-        ]
-      },
-      "data": {
-        "source": "The data is collected from professional medical board exams.",
-        "size": "The dataset contains 12,723 questions in English, 34,251 in simplified Chinese, and 14,123 in traditional Chinese. The total number of examples falls within the 10K to 100K range.",
-        "format": "JSON",
-        "annotation": "The answer labels are the correct answers from the professional exams. No additional annotation process is described."
-      },
-      "methodology": {
-        "methods": [
-          "The benchmark uses a sequential combination of a document retriever and a machine comprehension model. It includes both rule-based and neural methods.",
-          "The evaluation is a standard question-answering task, though the specific learning setting (e.g., zero-shot, few-shot, fine-tuning) is not explicitly defined."
-        ],
-        "metrics": [
-          "Accuracy"
-        ],
-        "calculation": "The overall score is the accuracy on the test set.",
-        "interpretation": "Higher accuracy indicates better performance. The best reported accuracies are 36.7% for English, 42.0% for traditional Chinese, and 70.1% for simplified Chinese questions.",
-        "baseline_results": "Original paper baselines: The best method reported achieves 36.7% accuracy on English, 42.0% on traditional Chinese, and 70.1% on simplified Chinese questions. Model names are not specified. EEE results: Yi 34B achieves a score of 0.656 (65.6%).",
-        "validation": "Not specified"
-      },
-      "ethical_and_legal_considerations": {
-        "privacy_and_anonymity": "Not specified",
-        "data_licensing": "Creative Commons Attribution 4.0",
-        "consent_procedures": "Not specified",
-        "compliance_with_regulations": "Not specified"
-      },
-      "possible_risks": [
-        {
-          "category": "Over- or under-reliance",
-          "description": [
-            "In AI-assisted decision-making tasks, reliance measures how much a person trusts (and potentially acts on) a model's output. Over-reliance occurs when a person puts too much trust in a model, accepting a model's output when the model's output is likely incorrect. Under-reliance is the opposite, where the person doesn't trust the model but should."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/over-or-under-reliance.html"
-        },
-        {
-          "category": "Unrepresentative data",
-          "description": [
-            "Unrepresentative data occurs when the training or fine-tuning data is not sufficiently representative of the underlying population or does not measure the phenomenon of interest. Synthetic data might not fully capture the complexity and nuances of real-world data. Causes include possible limitations in the seed data quality, biases in generation methods, or inadequate domain knowledge. Thus, AI models might struggle to generalize effectively to real-world scenarios."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/unrepresentative-data.html"
-        },
-        {
-          "category": "Uncertain data provenance",
-          "description": [
-            "Data provenance refers to the traceability of data (including synthetic data), which includes its ownership, origin, transformations, and generation. Proving that the data is the same as the original source with correct usage terms is difficult without standardized methods for verifying data sources or generation."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-provenance.html"
-        },
-        {
-          "category": "Data bias",
-          "description": [
-            "Historical and societal biases might be present in data that are used to train and fine-tune models. Biases can also be inherited from seed data or exacerbated by synthetic data generation methods."
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/data-bias.html"
-        },
-        {
-          "category": "Lack of data transparency",
-          "description": [
-            "Lack of data transparency might be due to insufficient documentation of training or tuning dataset details, including synthetic data generation. "
-          ],
-          "url": "https://www.ibm.com/docs/en/watsonx/saas?topic=SSYOK8/wsj/ai-risk-atlas/lack-of-data-transparency.html"
-        }
-      ],
-      "flagged_fields": {},
-      "missing_fields": [
-        "purpose_and_intended_users.out_of_scope_uses",
-        "methodology.validation",
-        "ethical_and_legal_considerations.privacy_and_anonymity",
-        "ethical_and_legal_considerations.consent_procedures",
-        "ethical_and_legal_considerations.compliance_with_regulations"
-      ],
-      "card_info": {
-        "created_at": "2026-03-17T13:23:29.822123",
-        "llm": "deepseek-ai/DeepSeek-V3.2"
-      }
-    }
-  },
-  "models": [
-    {
-      "model_id": "01-ai/yi-34b",
-      "name": "Yi 34B",
-      "developer": "01-ai",
-      "scores": {
-        "Mean win rate": 0.57,
-        "NarrativeQA": 0.782,
-        "NaturalQuestions (closed-book)": 0.443,
-        "OpenbookQA": 0.92,
-        "MMLU": 0.65,
-        "MATH": 0.375,
-        "GSM8K": 0.648,
-        "LegalBench": 0.618,
-        "MedQA": 0.656,
-        "WMT 2014": 0.172
-      }
-    },
-    {
-      "model_id": "01-ai/yi-6b",
-      "name": "Yi 6B",
-      "developer": "01-ai",
-      "scores": {
-        "Mean win rate": 0.253,
-        "NarrativeQA": 0.702,
-        "NaturalQuestions (closed-book)": 0.31,
-        "OpenbookQA": 0.8,
-        "MMLU": 0.53,
-        "MATH": 0.126,
-        "GSM8K": 0.375,
-        "LegalBench": 0.519,
-        "MedQA": 0.497,
-        "WMT 2014": 0.117
-      }
-    },
-    {
-      "model_id": "01-ai/yi-large-preview",
-      "name": "Yi Large Preview",
-      "developer": "01-ai",
-      "scores": {
-        "Mean win rate": 0.471,
-        "NarrativeQA": 0.373,
-        "NaturalQuestions (closed-book)": 0.428,
-        "OpenbookQA": 0.946,
-        "MMLU": 0.712,
-        "MATH": 0.712,
-        "GSM8K": 0.69,
-        "LegalBench": 0.519,
-        "MedQA": 0.66,
-        "WMT 2014": 0.176
-      }
-    },
-    {
-      "model_id": "AlephAlpha/luminous-base",
-      "name": "Luminous Base 13B",
-      "developer": "AlephAlpha",
-      "scores": {
-        "Mean win rate": 0.041,
-        "NarrativeQA": 0.633,
-        "NaturalQuestions (closed-book)": 0.197,
-        "OpenbookQA": 0.286,
-        "MMLU": 0.243,
-        "MATH": 0.026,
-        "GSM8K": 0.028,
-        "LegalBench": 0.332,
-        "MedQA": 0.26,
-        "WMT 2014": 0.066
-      }
-    },
-    {
-      "model_id": "AlephAlpha/luminous-extended",
-      "name": "Luminous Extended 30B",
-      "developer": "AlephAlpha",
-      "scores": {
-        "Mean win rate": 0.078,
-        "NarrativeQA": 0.684,
-        "NaturalQuestions (closed-book)": 0.253,
-        "OpenbookQA": 0.272,
-        "MMLU": 0.248,
-        "MATH": 0.04,
-        "GSM8K": 0.075,
-        "LegalBench": 0.421,
-        "MedQA": 0.276,
-        "WMT 2014": 0.083
-      }
-    },
-    {
-      "model_id": "AlephAlpha/luminous-supreme",
-      "name": "Luminous Supreme 70B",
-      "developer": "AlephAlpha",
-      "scores": {
-        "Mean win rate": 0.145,
-        "NarrativeQA": 0.743,
-        "NaturalQuestions (closed-book)": 0.299,
-        "OpenbookQA": 0.284,
-        "MMLU": 0.316,
-        "MATH": 0.078,
-        "GSM8K": 0.137,
-        "LegalBench": 0.452,
-        "MedQA": 0.276,
-        "WMT 2014": 0.102
-      }
-    },
-    {
-      "model_id": "ai21/j2-grande",
-      "name": "Jurassic-2 Grande 17B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.172,
-        "NarrativeQA": 0.744,
-        "NaturalQuestions (closed-book)": 0.35,
-        "OpenbookQA": 0.614,
-        "MMLU": 0.471,
-        "MATH": 0.064,
-        "GSM8K": 0.159,
-        "LegalBench": 0.468,
-        "MedQA": 0.39,
-        "WMT 2014": 0.102
-      }
-    },
-    {
-      "model_id": "ai21/j2-jumbo",
-      "name": "Jurassic-2 Jumbo 178B",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.215,
-        "NarrativeQA": 0.728,
-        "NaturalQuestions (closed-book)": 0.385,
-        "OpenbookQA": 0.688,
-        "MMLU": 0.483,
-        "MATH": 0.103,
-        "GSM8K": 0.239,
-        "LegalBench": 0.533,
-        "MedQA": 0.431,
-        "WMT 2014": 0.114
-      }
-    },
-    {
-      "model_id": "ai21/jamba-1.5-large",
-      "name": "Jamba 1.5 Large",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.637,
-        "NarrativeQA": 0.664,
-        "NaturalQuestions (closed-book)": 0.394,
-        "OpenbookQA": 0.948,
-        "MMLU": 0.683,
-        "MATH": 0.692,
-        "GSM8K": 0.846,
-        "LegalBench": 0.675,
-        "MedQA": 0.698,
-        "WMT 2014": 0.203
-      }
-    },
-    {
-      "model_id": "ai21/jamba-1.5-mini",
-      "name": "Jamba 1.5 Mini",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.414,
-        "NarrativeQA": 0.746,
-        "NaturalQuestions (closed-book)": 0.388,
-        "OpenbookQA": 0.89,
-        "MMLU": 0.582,
-        "MATH": 0.318,
-        "GSM8K": 0.691,
-        "LegalBench": 0.503,
-        "MedQA": 0.632,
-        "WMT 2014": 0.179
-      }
-    },
-    {
-      "model_id": "ai21/jamba-instruct",
-      "name": "Jamba Instruct",
-      "developer": "ai21",
-      "scores": {
-        "Mean win rate": 0.287,
-        "NarrativeQA": 0.658,
-        "NaturalQuestions (closed-book)": 0.384,
-        "OpenbookQA": 0.796,
-        "MMLU": 0.582,
-        "MATH": 0.38,
-        "GSM8K": 0.67,
-        "LegalBench": 0.54,
-        "MedQA": 0.519,
-        "WMT 2014": 0.164
-      }
-    },
-    {
-      "model_id": "allenai/olmo-7b",
-      "name": "OLMo 7B",
-      "developer": "allenai",
-      "scores": {
-        "Mean win rate": 0.052,
-        "NarrativeQA": 0.597,
-        "NaturalQuestions (closed-book)": 0.259,
-        "OpenbookQA": 0.222,
-        "MMLU": 0.305,
-        "MATH": 0.029,
-        "GSM8K": 0.044,
-        "LegalBench": 0.341,
-        "MedQA": 0.229,
-        "WMT 2014": 0.097
-      }
-    },
-    {
-      "model_id": "amazon/nova-lite-v1:0",
-      "name": "Amazon Nova Lite",
-      "developer": "amazon",
-      "scores": {
-        "Mean win rate": 0.708,
-        "NarrativeQA": 0.768,
-        "NaturalQuestions (closed-book)": 0.352,
-        "OpenbookQA": 0.928,
-        "MMLU": 0.693,
-        "MATH": 0.779,
-        "GSM8K": 0.829,
-        "LegalBench": 0.659,
-        "MedQA": 0.696,
-        "WMT 2014": 0.204
-      }
-    },
-    {
-      "model_id": "amazon/nova-micro-v1:0",
-      "name": "Amazon Nova Micro",
-      "developer": "amazon",
-      "scores": {
-        "Mean win rate": 0.524,
-        "NarrativeQA": 0.744,
-        "NaturalQuestions (closed-book)": 0.285,
-        "OpenbookQA": 0.888,
-        "MMLU": 0.64,
-        "MATH": 0.76,
-        "GSM8K": 0.794,
-        "LegalBench": 0.615,
-        "MedQA": 0.608,
-        "WMT 2014": 0.192
-      }
-    },
-    {
-      "model_id": "amazon/nova-pro-v1:0",
-      "name": "Amazon Nova Pro",
-      "developer": "amazon",
-      "scores": {
-        "Mean win rate": 0.885,
-        "NarrativeQA": 0.791,
-        "NaturalQuestions (closed-book)": 0.405,
-        "OpenbookQA": 0.96,
-        "MMLU": 0.758,
-        "MATH": 0.821,
-        "GSM8K": 0.87,
-        "LegalBench": 0.736,
-        "MedQA": 0.811,
-        "WMT 2014": 0.229
-      }
-    },
-    {
-      "model_id": "anthropic/claude-2.0",
-      "name": "Claude 2.0",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.489,
-        "NarrativeQA": 0.718,
-        "NaturalQuestions (closed-book)": 0.428,
-        "OpenbookQA": 0.862,
-        "MMLU": 0.639,
-        "MATH": 0.603,
-        "GSM8K": 0.583,
-        "LegalBench": 0.643,
-        "MedQA": 0.652,
-        "WMT 2014": 0.219
-      }
-    },
-    {
-      "model_id": "anthropic/claude-2.1",
-      "name": "Claude 2.1",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.437,
-        "NarrativeQA": 0.677,
-        "NaturalQuestions (closed-book)": 0.375,
-        "OpenbookQA": 0.872,
-        "MMLU": 0.643,
-        "MATH": 0.632,
-        "GSM8K": 0.604,
-        "LegalBench": 0.643,
-        "MedQA": 0.644,
-        "WMT 2014": 0.204
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-5-haiku-20241022",
-      "name": "Claude 3.5 Haiku 20241022",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.531,
-        "NarrativeQA": 0.763,
-        "NaturalQuestions (closed-book)": 0.344,
-        "OpenbookQA": 0.854,
-        "MMLU": 0.671,
-        "MATH": 0.872,
-        "GSM8K": 0.815,
-        "LegalBench": 0.631,
-        "MedQA": 0.722,
-        "WMT 2014": 0.135
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-5-sonnet-20240620",
-      "name": "Claude 3.5 Sonnet 20240620",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.885,
-        "NarrativeQA": 0.746,
-        "NaturalQuestions (closed-book)": 0.502,
-        "OpenbookQA": 0.972,
-        "MMLU": 0.799,
-        "MATH": 0.813,
-        "GSM8K": 0.949,
-        "LegalBench": 0.707,
-        "MedQA": 0.825,
-        "WMT 2014": 0.229
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-5-sonnet-20241022",
-      "name": "Claude 3.5 Sonnet 20241022",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.846,
-        "NarrativeQA": 0.77,
-        "NaturalQuestions (closed-book)": 0.467,
-        "OpenbookQA": 0.966,
-        "MMLU": 0.809,
-        "MATH": 0.904,
-        "GSM8K": 0.956,
-        "LegalBench": 0.647,
-        "MedQA": 0.859,
-        "WMT 2014": 0.226
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-haiku-20240307",
-      "name": "Claude 3 Haiku 20240307",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.263,
-        "NarrativeQA": 0.244,
-        "NaturalQuestions (closed-book)": 0.144,
-        "OpenbookQA": 0.838,
-        "MMLU": 0.662,
-        "MATH": 0.131,
-        "GSM8K": 0.699,
-        "LegalBench": 0.46,
-        "MedQA": 0.702,
-        "WMT 2014": 0.148
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-opus-20240229",
-      "name": "Claude 3 Opus 20240229",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.683,
-        "NarrativeQA": 0.351,
-        "NaturalQuestions (closed-book)": 0.441,
-        "OpenbookQA": 0.956,
-        "MMLU": 0.768,
-        "MATH": 0.76,
-        "GSM8K": 0.924,
-        "LegalBench": 0.662,
-        "MedQA": 0.775,
-        "WMT 2014": 0.24
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-sonnet-20240229",
-      "name": "Claude 3 Sonnet 20240229",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.377,
-        "NarrativeQA": 0.111,
-        "NaturalQuestions (closed-book)": 0.028,
-        "OpenbookQA": 0.918,
-        "MMLU": 0.652,
-        "MATH": 0.084,
-        "GSM8K": 0.907,
-        "LegalBench": 0.49,
-        "MedQA": 0.684,
-        "WMT 2014": 0.218
-      }
-    },
-    {
-      "model_id": "anthropic/claude-instant-1.2",
-      "name": "Claude Instant 1.2",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.399,
-        "NarrativeQA": 0.616,
-        "NaturalQuestions (closed-book)": 0.343,
-        "OpenbookQA": 0.844,
-        "MMLU": 0.631,
-        "MATH": 0.499,
-        "GSM8K": 0.721,
-        "LegalBench": 0.586,
-        "MedQA": 0.559,
-        "WMT 2014": 0.194
-      }
-    },
-    {
-      "model_id": "anthropic/claude-v1.3",
-      "name": "Anthropic Claude v1.3",
-      "developer": "Anthropic",
-      "scores": {
-        "Mean win rate": 0.518,
-        "NarrativeQA": 0.723,
-        "NaturalQuestions (closed-book)": 0.409,
-        "OpenbookQA": 0.908,
-        "MMLU": 0.631,
-        "MATH": 0.54,
-        "GSM8K": 0.784,
-        "LegalBench": 0.629,
-        "MedQA": 0.618,
-        "WMT 2014": 0.219
-      }
-    },
-    {
-      "model_id": "cohere/command",
-      "name": "Command",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.327,
-        "NarrativeQA": 0.749,
-        "NaturalQuestions (closed-book)": 0.391,
-        "OpenbookQA": 0.774,
-        "MMLU": 0.525,
-        "MATH": 0.236,
-        "GSM8K": 0.452,
-        "LegalBench": 0.578,
-        "MedQA": 0.445,
-        "WMT 2014": 0.088
-      }
-    },
-    {
-      "model_id": "cohere/command-light",
-      "name": "Command Light",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.105,
-        "NarrativeQA": 0.629,
-        "NaturalQuestions (closed-book)": 0.195,
-        "OpenbookQA": 0.398,
-        "MMLU": 0.386,
-        "MATH": 0.098,
-        "GSM8K": 0.149,
-        "LegalBench": 0.397,
-        "MedQA": 0.312,
-        "WMT 2014": 0.023
-      }
-    },
-    {
-      "model_id": "cohere/command-r",
-      "name": "Command R",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.299,
-        "NarrativeQA": 0.742,
-        "NaturalQuestions (closed-book)": 0.352,
-        "OpenbookQA": 0.782,
-        "MMLU": 0.567,
-        "MATH": 0.266,
-        "GSM8K": 0.551,
-        "LegalBench": 0.507,
-        "MedQA": 0.555,
-        "WMT 2014": 0.149
-      }
-    },
-    {
-      "model_id": "cohere/command-r-plus",
-      "name": "Command R Plus",
-      "developer": "cohere",
-      "scores": {
-        "Mean win rate": 0.441,
-        "NarrativeQA": 0.735,
-        "NaturalQuestions (closed-book)": 0.343,
-        "OpenbookQA": 0.828,
-        "MMLU": 0.59,
-        "MATH": 0.403,
-        "GSM8K": 0.738,
-        "LegalBench": 0.672,
-        "MedQA": 0.567,
-        "WMT 2014": 0.203
-      }
-    },
-    {
-      "model_id": "databricks/dbrx-instruct",
-      "name": "DBRX Instruct",
-      "developer": "databricks",
-      "scores": {
-        "Mean win rate": 0.289,
-        "NarrativeQA": 0.488,
-        "NaturalQuestions (closed-book)": 0.284,
-        "OpenbookQA": 0.91,
-        "MMLU": 0.643,
-        "MATH": 0.358,
-        "GSM8K": 0.671,
-        "LegalBench": 0.426,
-        "MedQA": 0.694,
-        "WMT 2014": 0.131
-      }
-    },
-    {
-      "model_id": "deepseek-ai/deepseek-llm-67b-chat",
-      "name": "DeepSeek LLM Chat 67B",
-      "developer": "deepseek-ai",
-      "scores": {
-        "Mean win rate": 0.488,
-        "NarrativeQA": 0.581,
-        "NaturalQuestions (closed-book)": 0.412,
-        "OpenbookQA": 0.88,
-        "MMLU": 0.641,
-        "MATH": 0.615,
-        "GSM8K": 0.795,
-        "LegalBench": 0.637,
-        "MedQA": 0.628,
-        "WMT 2014": 0.186
-      }
-    },
-    {
-      "model_id": "deepseek-ai/deepseek-v3",
-      "name": "DeepSeek v3",
-      "developer": "deepseek-ai",
-      "scores": {
-        "Mean win rate": 0.908,
-        "NarrativeQA": 0.796,
-        "NaturalQuestions (closed-book)": 0.467,
-        "OpenbookQA": 0.954,
-        "MMLU": 0.803,
-        "MATH": 0.912,
-        "GSM8K": 0.94,
-        "LegalBench": 0.718,
-        "MedQA": 0.809,
-        "WMT 2014": 0.209
-      }
-    },
-    {
-      "model_id": "google/gemini-1.0-pro-002",
-      "name": "Gemini 1.0 Pro 002",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.422,
-        "NarrativeQA": 0.751,
-        "NaturalQuestions (closed-book)": 0.391,
-        "OpenbookQA": 0.788,
-        "MMLU": 0.534,
-        "MATH": 0.665,
-        "GSM8K": 0.816,
-        "LegalBench": 0.475,
-        "MedQA": 0.483,
-        "WMT 2014": 0.194
-      }
-    },
-    {
-      "model_id": "google/gemini-1.5-flash-001",
-      "name": "Gemini 1.5 Flash 001",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.667,
-        "NarrativeQA": 0.783,
-        "NaturalQuestions (closed-book)": 0.332,
-        "OpenbookQA": 0.928,
-        "MMLU": 0.703,
-        "MATH": 0.753,
-        "GSM8K": 0.785,
-        "LegalBench": 0.661,
-        "MedQA": 0.68,
-        "WMT 2014": 0.225
-      }
-    },
-    {
-      "model_id": "google/gemini-1.5-flash-002",
-      "name": "Gemini 1.5 Flash 002",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.573,
-        "NarrativeQA": 0.746,
-        "NaturalQuestions (closed-book)": 0.323,
-        "OpenbookQA": 0.914,
-        "MMLU": 0.679,
-        "MATH": 0.908,
-        "GSM8K": 0.328,
-        "LegalBench": 0.67,
-        "MedQA": 0.656,
-        "WMT 2014": 0.212
-      }
-    },
-    {
-      "model_id": "google/gemini-1.5-pro-001",
-      "name": "Gemini 1.5 Pro 001",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.739,
-        "NarrativeQA": 0.783,
-        "NaturalQuestions (closed-book)": 0.378,
-        "OpenbookQA": 0.902,
-        "MMLU": 0.772,
-        "MATH": 0.825,
-        "GSM8K": 0.836,
-        "LegalBench": 0.757,
-        "MedQA": 0.692,
-        "WMT 2014": 0.189
-      }
-    },
-    {
-      "model_id": "google/gemini-1.5-pro-002",
-      "name": "Gemini 1.5 Pro 002",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.842,
-        "NarrativeQA": 0.756,
-        "NaturalQuestions (closed-book)": 0.455,
-        "OpenbookQA": 0.952,
-        "MMLU": 0.795,
-        "MATH": 0.92,
-        "GSM8K": 0.817,
-        "LegalBench": 0.747,
-        "MedQA": 0.771,
-        "WMT 2014": 0.231
-      }
-    },
-    {
-      "model_id": "google/gemini-2.0-flash-exp",
-      "name": "Gemini 2.0 Flash Experimental",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.813,
-        "NarrativeQA": 0.783,
-        "NaturalQuestions (closed-book)": 0.443,
-        "OpenbookQA": 0.946,
-        "MMLU": 0.717,
-        "MATH": 0.901,
-        "GSM8K": 0.946,
-        "LegalBench": 0.674,
-        "MedQA": 0.73,
-        "WMT 2014": 0.212
-      }
-    },
-    {
-      "model_id": "google/gemma-2-27b-it",
-      "name": "Gemma 2 Instruct 27B",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.675,
-        "NarrativeQA": 0.79,
-        "NaturalQuestions (closed-book)": 0.353,
-        "OpenbookQA": 0.918,
-        "MMLU": 0.664,
-        "MATH": 0.746,
-        "GSM8K": 0.812,
-        "LegalBench": 0.7,
-        "MedQA": 0.684,
-        "WMT 2014": 0.214
-      }
-    },
-    {
-      "model_id": "google/gemma-2-9b-it",
-      "name": "Gemma 2 Instruct 9B",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.562,
-        "NarrativeQA": 0.768,
-        "NaturalQuestions (closed-book)": 0.328,
-        "OpenbookQA": 0.91,
-        "MMLU": 0.645,
-        "MATH": 0.724,
-        "GSM8K": 0.762,
-        "LegalBench": 0.639,
-        "MedQA": 0.63,
-        "WMT 2014": 0.201
-      }
-    },
-    {
-      "model_id": "google/gemma-7b",
-      "name": "Gemma 7B",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.336,
-        "NarrativeQA": 0.752,
-        "NaturalQuestions (closed-book)": 0.336,
-        "OpenbookQA": 0.808,
-        "MMLU": 0.571,
-        "MATH": 0.5,
-        "GSM8K": 0.559,
-        "LegalBench": 0.581,
-        "MedQA": 0.513,
-        "WMT 2014": 0.187
-      }
-    },
-    {
-      "model_id": "google/text-bison@001",
-      "name": "PaLM-2 Bison",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.526,
-        "NarrativeQA": 0.718,
-        "NaturalQuestions (closed-book)": 0.39,
-        "OpenbookQA": 0.878,
-        "MMLU": 0.608,
-        "MATH": 0.421,
-        "GSM8K": 0.61,
-        "LegalBench": 0.645,
-        "MedQA": 0.547,
-        "WMT 2014": 0.241
-      }
-    },
-    {
-      "model_id": "google/text-unicorn@001",
-      "name": "PaLM-2 Unicorn",
-      "developer": "Google",
-      "scores": {
-        "Mean win rate": 0.644,
-        "NarrativeQA": 0.583,
-        "NaturalQuestions (closed-book)": 0.435,
-        "OpenbookQA": 0.938,
-        "MMLU": 0.702,
-        "MATH": 0.674,
-        "GSM8K": 0.831,
-        "LegalBench": 0.677,
-        "MedQA": 0.684,
-        "WMT 2014": 0.26
-      }
-    },
-    {
-      "model_id": "meta/LLaMA-65B",
-      "name": "LLaMA 65B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.345,
-        "NarrativeQA": 0.755,
-        "NaturalQuestions (closed-book)": 0.433,
-        "OpenbookQA": 0.754,
-        "MMLU": 0.584,
-        "MATH": 0.257,
-        "GSM8K": 0.489,
-        "LegalBench": 0.48,
-        "MedQA": 0.507,
-        "WMT 2014": 0.189
-      }
-    },
-    {
-      "model_id": "meta/llama-2-13b",
-      "name": "Llama 2 13B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.233,
-        "NarrativeQA": 0.741,
-        "NaturalQuestions (closed-book)": 0.371,
-        "OpenbookQA": 0.634,
-        "MMLU": 0.505,
-        "MATH": 0.102,
-        "GSM8K": 0.266,
-        "LegalBench": 0.591,
-        "MedQA": 0.392,
-        "WMT 2014": 0.167
-      }
-    },
-    {
-      "model_id": "meta/llama-2-70b",
-      "name": "Llama 2 70B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.482,
-        "NarrativeQA": 0.763,
-        "NaturalQuestions (closed-book)": 0.46,
-        "OpenbookQA": 0.838,
-        "MMLU": 0.58,
-        "MATH": 0.323,
-        "GSM8K": 0.567,
-        "LegalBench": 0.673,
-        "MedQA": 0.618,
-        "WMT 2014": 0.196
-      }
-    },
-    {
-      "model_id": "meta/llama-2-7b",
-      "name": "Llama 2 7B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.152,
-        "NarrativeQA": 0.686,
-        "NaturalQuestions (closed-book)": 0.333,
-        "OpenbookQA": 0.544,
-        "MMLU": 0.425,
-        "MATH": 0.097,
-        "GSM8K": 0.154,
-        "LegalBench": 0.502,
-        "MedQA": 0.392,
-        "WMT 2014": 0.144
-      }
-    },
-    {
-      "model_id": "meta/llama-3-70b",
-      "name": "Llama 3 70B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.793,
-        "NarrativeQA": 0.798,
-        "NaturalQuestions (closed-book)": 0.475,
-        "OpenbookQA": 0.934,
-        "MMLU": 0.695,
-        "MATH": 0.663,
-        "GSM8K": 0.805,
-        "LegalBench": 0.733,
-        "MedQA": 0.777,
-        "WMT 2014": 0.225
-      }
-    },
-    {
-      "model_id": "meta/llama-3-8b",
-      "name": "Llama 3 8B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.387,
-        "NarrativeQA": 0.754,
-        "NaturalQuestions (closed-book)": 0.378,
-        "OpenbookQA": 0.766,
-        "MMLU": 0.602,
-        "MATH": 0.391,
-        "GSM8K": 0.499,
-        "LegalBench": 0.637,
-        "MedQA": 0.581,
-        "WMT 2014": 0.183
-      }
-    },
-    {
-      "model_id": "meta/llama-3.1-405b-instruct-turbo",
-      "name": "Llama 3.1 Instruct Turbo 405B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.854,
-        "NarrativeQA": 0.749,
-        "NaturalQuestions (closed-book)": 0.456,
-        "OpenbookQA": 0.94,
-        "MMLU": 0.759,
-        "MATH": 0.827,
-        "GSM8K": 0.949,
-        "LegalBench": 0.707,
-        "MedQA": 0.805,
-        "WMT 2014": 0.238
-      }
-    },
-    {
-      "model_id": "meta/llama-3.1-70b-instruct-turbo",
-      "name": "Llama 3.1 Instruct Turbo 70B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.808,
-        "NarrativeQA": 0.772,
-        "NaturalQuestions (closed-book)": 0.452,
-        "OpenbookQA": 0.938,
-        "MMLU": 0.709,
-        "MATH": 0.783,
-        "GSM8K": 0.938,
-        "LegalBench": 0.687,
-        "MedQA": 0.769,
-        "WMT 2014": 0.223
-      }
-    },
-    {
-      "model_id": "meta/llama-3.1-8b-instruct-turbo",
-      "name": "Llama 3.1 Instruct Turbo 8B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.303,
-        "NarrativeQA": 0.756,
-        "NaturalQuestions (closed-book)": 0.209,
-        "OpenbookQA": 0.74,
-        "MMLU": 0.5,
-        "MATH": 0.703,
-        "GSM8K": 0.798,
-        "LegalBench": 0.342,
-        "MedQA": 0.245,
-        "WMT 2014": 0.181
-      }
-    },
-    {
-      "model_id": "meta/llama-3.2-11b-vision-instruct-turbo",
-      "name": "Llama 3.2 Vision Instruct Turbo 11B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.325,
-        "NarrativeQA": 0.756,
-        "NaturalQuestions (closed-book)": 0.234,
-        "OpenbookQA": 0.724,
-        "MMLU": 0.511,
-        "MATH": 0.739,
-        "GSM8K": 0.823,
-        "LegalBench": 0.435,
-        "MedQA": 0.27,
-        "WMT 2014": 0.179
-      }
-    },
-    {
-      "model_id": "meta/llama-3.2-90b-vision-instruct-turbo",
-      "name": "Llama 3.2 Vision Instruct Turbo 90B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.819,
-        "NarrativeQA": 0.777,
-        "NaturalQuestions (closed-book)": 0.457,
-        "OpenbookQA": 0.942,
-        "MMLU": 0.703,
-        "MATH": 0.791,
-        "GSM8K": 0.936,
-        "LegalBench": 0.68,
-        "MedQA": 0.769,
-        "WMT 2014": 0.224
-      }
-    },
-    {
-      "model_id": "meta/llama-3.3-70b-instruct-turbo",
-      "name": "Llama 3.3 Instruct Turbo 70B",
-      "developer": "Meta",
-      "scores": {
-        "Mean win rate": 0.812,
-        "NarrativeQA": 0.791,
-        "NaturalQuestions (closed-book)": 0.431,
-        "OpenbookQA": 0.928,
-        "MMLU": 0.7,
-        "MATH": 0.808,
-        "GSM8K": 0.942,
-        "LegalBench": 0.725,
-        "MedQA": 0.761,
-        "WMT 2014": 0.219
-      }
-    },
-    {
-      "model_id": "microsoft/phi-2",
-      "name": "Phi-2",
-      "developer": "microsoft",
-      "scores": {
-        "Mean win rate": 0.169,
-        "NarrativeQA": 0.703,
-        "NaturalQuestions (closed-book)": 0.155,
-        "OpenbookQA": 0.798,
-        "MMLU": 0.518,
-        "MATH": 0.255,
-        "GSM8K": 0.581,
-        "LegalBench": 0.334,
-        "MedQA": 0.41,
-        "WMT 2014": 0.038
-      }
-    },
-    {
-      "model_id": "microsoft/phi-3-medium-4k-instruct",
-      "name": "Phi-3 14B",
-      "developer": "microsoft",
-      "scores": {
-        "Mean win rate": 0.509,
-        "NarrativeQA": 0.724,
-        "NaturalQuestions (closed-book)": 0.278,
-        "OpenbookQA": 0.916,
-        "MMLU": 0.675,
-        "MATH": 0.611,
-        "GSM8K": 0.878,
-        "LegalBench": 0.593,
-        "MedQA": 0.696,
-        "WMT 2014": 0.17
-      }
-    },
-    {
-      "model_id": "microsoft/phi-3-small-8k-instruct",
-      "name": "Phi-3 7B",
-      "developer": "microsoft",
-      "scores": {
-        "Mean win rate": 0.473,
-        "NarrativeQA": 0.754,
-        "NaturalQuestions (closed-book)": 0.324,
-        "OpenbookQA": 0.912,
-        "MMLU": 0.659,
-        "MATH": 0.703,
-        "GSM8K": -1,
-        "LegalBench": 0.584,
-        "MedQA": 0.672,
-        "WMT 2014": 0.154
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-7b-instruct-v0.3",
-      "name": "Mistral Instruct v0.3 7B",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.196,
-        "NarrativeQA": 0.716,
-        "NaturalQuestions (closed-book)": 0.253,
-        "OpenbookQA": 0.79,
-        "MMLU": 0.51,
-        "MATH": 0.289,
-        "GSM8K": 0.538,
-        "LegalBench": 0.331,
-        "MedQA": 0.517,
-        "WMT 2014": 0.142
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-7b-v0.1",
-      "name": "Mistral v0.1 7B",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.292,
-        "NarrativeQA": 0.716,
-        "NaturalQuestions (closed-book)": 0.367,
-        "OpenbookQA": 0.776,
-        "MMLU": 0.584,
-        "MATH": 0.297,
-        "GSM8K": 0.377,
-        "LegalBench": 0.58,
-        "MedQA": 0.525,
-        "WMT 2014": 0.16
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-large-2402",
-      "name": "Mistral Large 2402",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.328,
-        "NarrativeQA": 0.454,
-        "NaturalQuestions (closed-book)": 0.311,
-        "OpenbookQA": 0.894,
-        "MMLU": 0.638,
-        "MATH": 0.75,
-        "GSM8K": 0.694,
-        "LegalBench": 0.479,
-        "MedQA": 0.499,
-        "WMT 2014": 0.182
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-large-2407",
-      "name": "Mistral Large 2 2407",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.744,
-        "NarrativeQA": 0.779,
-        "NaturalQuestions (closed-book)": 0.453,
-        "OpenbookQA": 0.932,
-        "MMLU": 0.725,
-        "MATH": 0.677,
-        "GSM8K": 0.912,
-        "LegalBench": 0.646,
-        "MedQA": 0.775,
-        "WMT 2014": 0.192
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-medium-2312",
-      "name": "Mistral Medium 2312",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.268,
-        "NarrativeQA": 0.449,
-        "NaturalQuestions (closed-book)": 0.29,
-        "OpenbookQA": 0.83,
-        "MMLU": 0.618,
-        "MATH": 0.565,
-        "GSM8K": 0.706,
-        "LegalBench": 0.452,
-        "MedQA": 0.61,
-        "WMT 2014": 0.169
-      }
-    },
-    {
-      "model_id": "mistralai/mistral-small-2402",
-      "name": "Mistral Small 2402",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.288,
-        "NarrativeQA": 0.519,
-        "NaturalQuestions (closed-book)": 0.304,
-        "OpenbookQA": 0.862,
-        "MMLU": 0.593,
-        "MATH": 0.621,
-        "GSM8K": 0.734,
-        "LegalBench": 0.389,
-        "MedQA": 0.616,
-        "WMT 2014": 0.169
-      }
-    },
-    {
-      "model_id": "mistralai/mixtral-8x22b",
-      "name": "Mixtral 8x22B",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.705,
-        "NarrativeQA": 0.779,
-        "NaturalQuestions (closed-book)": 0.478,
-        "OpenbookQA": 0.882,
-        "MMLU": 0.701,
-        "MATH": 0.656,
-        "GSM8K": 0.8,
-        "LegalBench": 0.708,
-        "MedQA": 0.704,
-        "WMT 2014": 0.209
-      }
-    },
-    {
-      "model_id": "mistralai/mixtral-8x7b-32kseqlen",
-      "name": "Mixtral 8x7B 32K seqlen",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.51,
-        "NarrativeQA": 0.767,
-        "NaturalQuestions (closed-book)": 0.427,
-        "OpenbookQA": 0.868,
-        "MMLU": 0.649,
-        "MATH": 0.494,
-        "GSM8K": 0.622,
-        "LegalBench": 0.63,
-        "MedQA": 0.652,
-        "WMT 2014": 0.19
-      }
-    },
-    {
-      "model_id": "mistralai/open-mistral-nemo-2407",
-      "name": "Mistral NeMo 2402",
-      "developer": "mistralai",
-      "scores": {
-        "Mean win rate": 0.333,
-        "NarrativeQA": 0.731,
-        "NaturalQuestions (closed-book)": 0.265,
-        "OpenbookQA": 0.822,
-        "MMLU": 0.604,
-        "MATH": 0.668,
-        "GSM8K": 0.782,
-        "LegalBench": 0.415,
-        "MedQA": 0.59,
-        "WMT 2014": 0.177
-      }
-    },
-    {
-      "model_id": "openai/gpt-3.5-turbo-0613",
-      "name": "GPT-3.5 Turbo 0613",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.358,
-        "NarrativeQA": 0.655,
-        "NaturalQuestions (closed-book)": 0.335,
-        "OpenbookQA": 0.838,
-        "MMLU": 0.614,
-        "MATH": 0.667,
-        "GSM8K": 0.501,
-        "LegalBench": 0.528,
-        "MedQA": 0.622,
-        "WMT 2014": 0.187
-      }
-    },
-    {
-      "model_id": "openai/gpt-4-0613",
-      "name": "GPT-4 0613",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.867,
-        "NarrativeQA": 0.768,
-        "NaturalQuestions (closed-book)": 0.457,
-        "OpenbookQA": 0.96,
-        "MMLU": 0.735,
-        "MATH": 0.802,
-        "GSM8K": 0.932,
-        "LegalBench": 0.713,
-        "MedQA": 0.815,
-        "WMT 2014": 0.211
-      }
-    },
-    {
-      "model_id": "openai/gpt-4-1106-preview",
-      "name": "GPT-4 Turbo 1106 preview",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.698,
-        "NarrativeQA": 0.727,
-        "NaturalQuestions (closed-book)": 0.435,
-        "OpenbookQA": 0.95,
-        "MMLU": 0.699,
-        "MATH": 0.857,
-        "GSM8K": 0.668,
-        "LegalBench": 0.626,
-        "MedQA": 0.817,
-        "WMT 2014": 0.205
-      }
-    },
-    {
-      "model_id": "openai/gpt-4-turbo-2024-04-09",
-      "name": "GPT-4 Turbo 2024-04-09",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.864,
-        "NarrativeQA": 0.761,
-        "NaturalQuestions (closed-book)": 0.482,
-        "OpenbookQA": 0.97,
-        "MMLU": 0.711,
-        "MATH": 0.833,
-        "GSM8K": 0.824,
-        "LegalBench": 0.727,
-        "MedQA": 0.783,
-        "WMT 2014": 0.218
-      }
-    },
-    {
-      "model_id": "openai/gpt-4o-2024-05-13",
-      "name": "GPT-4o 2024-05-13",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.938,
-        "NarrativeQA": 0.804,
-        "NaturalQuestions (closed-book)": 0.501,
-        "OpenbookQA": 0.966,
-        "MMLU": 0.748,
-        "MATH": 0.829,
-        "GSM8K": 0.905,
-        "LegalBench": 0.733,
-        "MedQA": 0.857,
-        "WMT 2014": 0.231
-      }
-    },
-    {
-      "model_id": "openai/gpt-4o-2024-08-06",
-      "name": "GPT-4o 2024-08-06",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.928,
-        "NarrativeQA": 0.795,
-        "NaturalQuestions (closed-book)": 0.496,
-        "OpenbookQA": 0.968,
-        "MMLU": 0.738,
-        "MATH": 0.853,
-        "GSM8K": 0.909,
-        "LegalBench": 0.721,
-        "MedQA": 0.863,
-        "WMT 2014": 0.225
-      }
-    },
-    {
-      "model_id": "openai/gpt-4o-mini-2024-07-18",
-      "name": "GPT-4o mini 2024-07-18",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.701,
-        "NarrativeQA": 0.768,
-        "NaturalQuestions (closed-book)": 0.386,
-        "OpenbookQA": 0.92,
-        "MMLU": 0.668,
-        "MATH": 0.802,
-        "GSM8K": 0.843,
-        "LegalBench": 0.653,
-        "MedQA": 0.748,
-        "WMT 2014": 0.206
-      }
-    },
-    {
-      "model_id": "openai/text-davinci-002",
-      "name": "GPT-3.5 text-davinci-002",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.336,
-        "NarrativeQA": 0.719,
-        "NaturalQuestions (closed-book)": 0.394,
-        "OpenbookQA": 0.796,
-        "MMLU": 0.568,
-        "MATH": 0.428,
-        "GSM8K": 0.479,
-        "LegalBench": 0.58,
-        "MedQA": 0.525,
-        "WMT 2014": 0.174
-      }
-    },
-    {
-      "model_id": "openai/text-davinci-003",
-      "name": "GPT-3.5 text-davinci-003",
-      "developer": "OpenAI",
-      "scores": {
-        "Mean win rate": 0.439,
-        "NarrativeQA": 0.731,
-        "NaturalQuestions (closed-book)": 0.413,
-        "OpenbookQA": 0.828,
-        "MMLU": 0.555,
-        "MATH": 0.449,
-        "GSM8K": 0.615,
-        "LegalBench": 0.622,
-        "MedQA": 0.531,
-        "WMT 2014": 0.191
-      }
-    },
-    {
-      "model_id": "qwen/qwen1.5-110b-chat",
-      "name": "Qwen1.5 Chat 110B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.55,
-        "NarrativeQA": 0.721,
-        "NaturalQuestions (closed-book)": 0.35,
-        "OpenbookQA": 0.922,
-        "MMLU": 0.704,
-        "MATH": 0.568,
-        "GSM8K": 0.815,
-        "LegalBench": 0.624,
-        "MedQA": 0.64,
-        "WMT 2014": 0.192
-      }
-    },
-    {
-      "model_id": "qwen/qwen1.5-14b",
-      "name": "Qwen1.5 14B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.425,
-        "NarrativeQA": 0.711,
-        "NaturalQuestions (closed-book)": 0.3,
-        "OpenbookQA": 0.862,
-        "MMLU": 0.626,
-        "MATH": 0.686,
-        "GSM8K": 0.693,
-        "LegalBench": 0.593,
-        "MedQA": 0.515,
-        "WMT 2014": 0.178
-      }
-    },
-    {
-      "model_id": "qwen/qwen1.5-32b",
-      "name": "Qwen1.5 32B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.546,
-        "NarrativeQA": 0.589,
-        "NaturalQuestions (closed-book)": 0.353,
-        "OpenbookQA": 0.932,
-        "MMLU": 0.628,
-        "MATH": 0.733,
-        "GSM8K": 0.773,
-        "LegalBench": 0.636,
-        "MedQA": 0.656,
-        "WMT 2014": 0.193
-      }
-    },
-    {
-      "model_id": "qwen/qwen1.5-72b",
-      "name": "Qwen1.5 72B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.608,
-        "NarrativeQA": 0.601,
-        "NaturalQuestions (closed-book)": 0.417,
-        "OpenbookQA": 0.93,
-        "MMLU": 0.647,
-        "MATH": 0.683,
-        "GSM8K": 0.799,
-        "LegalBench": 0.694,
-        "MedQA": 0.67,
-        "WMT 2014": 0.201
-      }
-    },
-    {
-      "model_id": "qwen/qwen1.5-7b",
-      "name": "Qwen1.5 7B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.275,
-        "NarrativeQA": 0.448,
-        "NaturalQuestions (closed-book)": 0.27,
-        "OpenbookQA": 0.806,
-        "MMLU": 0.569,
-        "MATH": 0.561,
-        "GSM8K": 0.6,
-        "LegalBench": 0.523,
-        "MedQA": 0.479,
-        "WMT 2014": 0.153
-      }
-    },
-    {
-      "model_id": "qwen/qwen2-72b-instruct",
-      "name": "Qwen2 Instruct 72B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.77,
-        "NarrativeQA": 0.727,
-        "NaturalQuestions (closed-book)": 0.39,
-        "OpenbookQA": 0.954,
-        "MMLU": 0.769,
-        "MATH": 0.79,
-        "GSM8K": 0.92,
-        "LegalBench": 0.712,
-        "MedQA": 0.746,
-        "WMT 2014": 0.207
-      }
-    },
-    {
-      "model_id": "qwen/qwen2.5-72b-instruct-turbo",
-      "name": "Qwen2.5 Instruct Turbo 72B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.745,
-        "NarrativeQA": 0.745,
-        "NaturalQuestions (closed-book)": 0.359,
-        "OpenbookQA": 0.962,
-        "MMLU": 0.77,
-        "MATH": 0.884,
-        "GSM8K": 0.9,
-        "LegalBench": 0.74,
-        "MedQA": 0.753,
-        "WMT 2014": 0.207
-      }
-    },
-    {
-      "model_id": "qwen/qwen2.5-7b-instruct-turbo",
-      "name": "Qwen2.5 Instruct Turbo 7B",
-      "developer": "qwen",
-      "scores": {
-        "Mean win rate": 0.488,
-        "NarrativeQA": 0.742,
-        "NaturalQuestions (closed-book)": 0.205,
-        "OpenbookQA": 0.862,
-        "MMLU": 0.658,
-        "MATH": 0.835,
-        "GSM8K": 0.83,
-        "LegalBench": 0.632,
-        "MedQA": 0.6,
-        "WMT 2014": 0.155
-      }
-    },
-    {
-      "model_id": "snowflake/snowflake-arctic-instruct",
-      "name": "Arctic Instruct",
-      "developer": "snowflake",
-      "scores": {
-        "Mean win rate": 0.338,
-        "NarrativeQA": 0.654,
-        "NaturalQuestions (closed-book)": 0.39,
-        "OpenbookQA": 0.828,
-        "MMLU": 0.575,
-        "MATH": 0.519,
-        "GSM8K": 0.768,
-        "LegalBench": 0.588,
-        "MedQA": 0.581,
-        "WMT 2014": 0.172
-      }
-    },
-    {
-      "model_id": "tiiuae/falcon-40b",
-      "name": "Falcon 40B",
-      "developer": "tiiuae",
-      "scores": {
-        "Mean win rate": 0.217,
-        "NarrativeQA": 0.671,
-        "NaturalQuestions (closed-book)": 0.392,
-        "OpenbookQA": 0.662,
-        "MMLU": 0.507,
-        "MATH": 0.128,
-        "GSM8K": 0.267,
-        "LegalBench": 0.442,
-        "MedQA": 0.419,
-        "WMT 2014": 0.162
-      }
-    },
-    {
-      "model_id": "tiiuae/falcon-7b",
-      "name": "Falcon 7B",
-      "developer": "tiiuae",
-      "scores": {
-        "Mean win rate": 0.064,
-        "NarrativeQA": 0.621,
-        "NaturalQuestions (closed-book)": 0.285,
-        "OpenbookQA": 0.26,
-        "MMLU": 0.288,
-        "MATH": 0.044,
-        "GSM8K": 0.055,
-        "LegalBench": 0.346,
-        "MedQA": 0.254,
-        "WMT 2014": 0.094
-      }
-    },
-    {
-      "model_id": "upstage/solar-pro-241126",
-      "name": "Solar Pro",
-      "developer": "upstage",
-      "scores": {
-        "Mean win rate": 0.602,
-        "NarrativeQA": 0.753,
-        "NaturalQuestions (closed-book)": 0.297,
-        "OpenbookQA": 0.922,
-        "MMLU": 0.679,
-        "MATH": 0.567,
-        "GSM8K": 0.871,
-        "LegalBench": 0.67,
-        "MedQA": 0.698,
-        "WMT 2014": 0.169
-      }
-    },
-    {
-      "model_id": "writer/palmyra-x-004",
-      "name": "Palmyra-X-004",
-      "developer": "writer",
-      "scores": {
-        "Mean win rate": 0.808,
-        "NarrativeQA": 0.773,
-        "NaturalQuestions (closed-book)": 0.457,
-        "OpenbookQA": 0.926,
-        "MMLU": 0.739,
-        "MATH": 0.767,
-        "GSM8K": 0.905,
-        "LegalBench": 0.73,
-        "MedQA": 0.775,
-        "WMT 2014": 0.203
-      }
-    },
-    {
-      "model_id": "writer/palmyra-x-v2",
-      "name": "Palmyra X V2 33B",
-      "developer": "writer",
-      "scores": {
-        "Mean win rate": 0.589,
-        "NarrativeQA": 0.752,
-        "NaturalQuestions (closed-book)": 0.428,
-        "OpenbookQA": 0.878,
-        "MMLU": 0.621,
-        "MATH": 0.58,
-        "GSM8K": 0.735,
-        "LegalBench": 0.644,
-        "MedQA": 0.598,
-        "WMT 2014": 0.239
-      }
-    },
-    {
-      "model_id": "writer/palmyra-x-v3",
-      "name": "Palmyra X V3 72B",
-      "developer": "writer",
-      "scores": {
-        "Mean win rate": 0.679,
-        "NarrativeQA": 0.706,
-        "NaturalQuestions (closed-book)": 0.407,
-        "OpenbookQA": 0.938,
-        "MMLU": 0.702,
-        "MATH": 0.723,
-        "GSM8K": 0.831,
-        "LegalBench": 0.709,
-        "MedQA": 0.684,
-        "WMT 2014": 0.262
-      }
-    }
-  ]
-}

data/benchmarks/la_leaderboard.json

@@ -1,44 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "Qwen/Qwen2.5-7B",
-      "name": "Qwen2.5-7B",
-      "developer": "Qwen",
-      "scores": {
-        "la_leaderboard": 27.61
-      }
-    },
-    {
-      "model_id": "google/gemma-2-9b-it",
-      "name": "Gemma 2 Instruct 9B",
-      "developer": "Google",
-      "scores": {
-        "la_leaderboard": 33.62
-      }
-    },
-    {
-      "model_id": "meta-llama/Meta-Llama-3.1-8B",
-      "name": "Meta Llama 3.1 8B",
-      "developer": "unknown",
-      "scores": {
-        "la_leaderboard": 27.04
-      }
-    },
-    {
-      "model_id": "meta-llama/Meta-Llama-3.1-8B-Instruct",
-      "name": "Meta Llama 3.1 8B Instruct",
-      "developer": "unknown",
-      "scores": {
-        "la_leaderboard": 30.23
-      }
-    },
-    {
-      "model_id": "utter-project/EuroLLM-9B",
-      "name": "EuroLLM 9B",
-      "developer": "unknown",
-      "scores": {
-        "la_leaderboard": 25.87
-      }
-    }
-  ]
-}

data/benchmarks/livecodebenchpro.json

@@ -1,274 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "alibaba/qwen3-235b-a22b-thinking-2507",
-      "name": "qwen3-235b-a22b-thinking-2507",
-      "developer": "Alibaba",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.1267605633802817,
-        "Easy Problems": 0.7605633802816901
-      }
-    },
-    {
-      "model_id": "alibaba/qwen3-30b-a3b",
-      "name": "qwen3-30b-a3b",
-      "developer": "Alibaba",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.028169014084507043,
-        "Easy Problems": 0.5774647887323944
-      }
-    },
-    {
-      "model_id": "alibaba/qwen3-max",
-      "name": "alibaba/qwen3-max",
-      "developer": "Alibaba",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.04225352112676056,
-        "Easy Problems": 0.36619718309859156
-      }
-    },
-    {
-      "model_id": "alibaba/qwen3-next-80b-a3b-thinking",
-      "name": "qwen3-next-80b-a3b-thinking",
-      "developer": "Alibaba",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.14084507042253522,
-        "Easy Problems": 0.7464788732394366
-      }
-    },
-    {
-      "model_id": "aliyun/qwen3-next-80b-a3b-thinking",
-      "name": "qwen3-next-80b-a3b-thinking",
-      "developer": "aliyun",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0704,
-        "Easy Problems": 0.6901
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3-7-sonnet-20250219",
-      "name": "claude-3-7-sonnet-20250219",
-      "developer": "Anthropic",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0,
-        "Easy Problems": 0.28169014084507044
-      }
-    },
-    {
-      "model_id": "anthropic/claude-3.7-sonnet",
-      "name": "anthropic/claude-3.7-sonnet",
-      "developer": "Anthropic",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.014084507042253521,
-        "Easy Problems": 0.15492957746478872
-      }
-    },
-    {
-      "model_id": "anthropic/claude-sonnet-4-5-20250929",
-      "name": "claude-sonnet-4-5-20250929",
-      "developer": "Anthropic",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0,
-        "Easy Problems": 0.5352
-      }
-    },
-    {
-      "model_id": "ark/ep-20250603132404-cgpjm",
-      "name": "ep-20250603132404-cgpjm",
-      "developer": "ark",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0141,
-        "Easy Problems": 0.507
-      }
-    },
-    {
-      "model_id": "bytedance/doubao-seed-1-6-thinking-250615",
-      "name": "doubao-seed-1-6-thinking-250615",
-      "developer": "ByteDance",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.07042253521126761,
-        "Easy Problems": 0.5774647887323944
-      }
-    },
-    {
-      "model_id": "deepseek/chat-v3-0324",
-      "name": "deepseek/chat-v3-0324",
-      "developer": "DeepSeek",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0,
-        "Easy Problems": 0.19718309859154928
-      }
-    },
-    {
-      "model_id": "deepseek/ep-20250214004308-p7n89",
-      "name": "ep-20250214004308-p7n89",
-      "developer": "DeepSeek",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.014084507042253521,
-        "Easy Problems": 0.4225352112676056
-      }
-    },
-    {
-      "model_id": "deepseek/ep-20250228232227-z44x5",
-      "name": "ep-20250228232227-z44x5",
-      "developer": "DeepSeek",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0,
-        "Easy Problems": 0.1267605633802817
-      }
-    },
-    {
-      "model_id": "deepseek/ep-20250603132404-cgpjm",
-      "name": "ep-20250603132404-cgpjm",
-      "developer": "DeepSeek",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.08450704225352113,
-        "Easy Problems": 0.5774647887323944
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-flash",
-      "name": "Gemini 2.5 Flash",
-      "developer": "Google",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.028169014084507043,
-        "Easy Problems": 0.38028169014084506
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-pro",
-      "name": "Gemini 2.5 Pro",
-      "developer": "Google",
-      "scores": {
-        "Hard Problems": 0.014084507042253521,
-        "Medium Problems": 0.2112676056338028,
-        "Easy Problems": 0.7183098591549296
-      }
-    },
-    {
-      "model_id": "kuaishou/kwaipilot-40b-0604",
-      "name": "kwaipilot-40b-0604",
-      "developer": "Kuaishou",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.07042253521126761,
-        "Easy Problems": 0.056338028169014086
-      }
-    },
-    {
-      "model_id": "meta/llama-4-maverick",
-      "name": "meta/llama-4-maverick",
-      "developer": "Meta",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0,
-        "Easy Problems": 0.09859154929577464
-      }
-    },
-    {
-      "model_id": "openai/gpt-4.1",
-      "name": "openai/gpt-4.1",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0,
-        "Easy Problems": 0.19718309859154928
-      }
-    },
-    {
-      "model_id": "openai/gpt-4o-2024-11-20",
-      "name": "GPT-4o 2024-11-20",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.0,
-        "Easy Problems": 0.07042253521126761
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-2025-08-07",
-      "name": "gpt-5-2025-08-07",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.0423,
-        "Medium Problems": 0.4085,
-        "Easy Problems": 0.9014
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-2025-12-11",
-      "name": "gpt-5.2-2025-12-11",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.1594,
-        "Medium Problems": 0.5211,
-        "Easy Problems": 0.9014
-      }
-    },
-    {
-      "model_id": "openai/gpt-oss-120b",
-      "name": "GPT-OSS-120B",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.11267605633802817,
-        "Easy Problems": 0.6619718309859155
-      }
-    },
-    {
-      "model_id": "openai/gpt-oss-20b",
-      "name": "GPT-OSS-20B",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.056338028169014086,
-        "Easy Problems": 0.5070422535211268
-      }
-    },
-    {
-      "model_id": "openai/o3-2025-04-16",
-      "name": "o3-2025-04-16",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.22535211267605634,
-        "Easy Problems": 0.7183098591549296
-      }
-    },
-    {
-      "model_id": "openai/o4-mini-2025-04-16",
-      "name": "o4-mini-2025-04-16",
-      "developer": "OpenAI",
-      "scores": {
-        "Hard Problems": 0.0143,
-        "Medium Problems": 0.2923,
-        "Easy Problems": 0.8571
-      }
-    },
-    {
-      "model_id": "z-ai/glm-4.5",
-      "name": "z-ai/glm-4.5",
-      "developer": "Z.ai",
-      "scores": {
-        "Hard Problems": 0.0,
-        "Medium Problems": 0.028169014084507043,
-        "Easy Problems": 0.1267605633802817
-      }
-    }
-  ]
-}

data/benchmarks/swe-bench.json

@@ -1,28 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/claude-opus-4-5",
-      "name": "claude-opus-4-5",
-      "developer": "Anthropic",
-      "scores": {
-        "swe-bench": 0.65
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro-preview",
-      "name": "gemini-3-pro-preview",
-      "developer": "Google",
-      "scores": {
-        "swe-bench": 0.71
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-2025-12-11",
-      "name": "gpt-5.2-2025-12-11",
-      "developer": "OpenAI",
-      "scores": {
-        "swe-bench": 0.57
-      }
-    }
-  ]
-}

data/benchmarks/tau-bench-2_airline.json

@@ -1,28 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/claude-opus-4-5",
-      "name": "claude-opus-4-5",
-      "developer": "Anthropic",
-      "scores": {
-        "tau-bench-2/airline": 0.66
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro-preview",
-      "name": "gemini-3-pro-preview",
-      "developer": "Google",
-      "scores": {
-        "tau-bench-2/airline": 0.62
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-2025-12-11",
-      "name": "gpt-5.2-2025-12-11",
-      "developer": "OpenAI",
-      "scores": {
-        "tau-bench-2/airline": 0.54
-      }
-    }
-  ]
-}

data/benchmarks/tau-bench-2_retail.json

@@ -1,28 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/claude-opus-4-5",
-      "name": "claude-opus-4-5",
-      "developer": "Anthropic",
-      "scores": {
-        "tau-bench-2/retail": 0.78
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro-preview",
-      "name": "gemini-3-pro-preview",
-      "developer": "Google",
-      "scores": {
-        "tau-bench-2/retail": 0.7576
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-2025-12-11",
-      "name": "gpt-5.2-2025-12-11",
-      "developer": "OpenAI",
-      "scores": {
-        "tau-bench-2/retail": 0.68
-      }
-    }
-  ]
-}

data/benchmarks/tau-bench-2_telecom.json

@@ -1,28 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "anthropic/claude-opus-4-5",
-      "name": "claude-opus-4-5",
-      "developer": "Anthropic",
-      "scores": {
-        "tau-bench-2/telecom": 0.84
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro-preview",
-      "name": "gemini-3-pro-preview",
-      "developer": "Google",
-      "scores": {
-        "tau-bench-2/telecom": 0.73
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-2025-12-11",
-      "name": "gpt-5.2-2025-12-11",
-      "developer": "OpenAI",
-      "scores": {
-        "tau-bench-2/telecom": 0.5354
-      }
-    }
-  ]
-}

data/benchmarks/terminal-bench-2.0.json

@@ -1,300 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "alibaba/qwen-3-coder-480b",
-      "name": "Qwen 3 Coder 480B",
-      "developer": "Alibaba",
-      "scores": {
-        "terminal-bench-2.0": 23.9
-      }
-    },
-    {
-      "model_id": "anthropic/claude-haiku-4.5",
-      "name": "Claude Haiku 4.5",
-      "developer": "Anthropic",
-      "scores": {
-        "terminal-bench-2.0": 35.5
-      }
-    },
-    {
-      "model_id": "anthropic/claude-opus-4.1",
-      "name": "Claude Opus 4.1",
-      "developer": "Anthropic",
-      "scores": {
-        "terminal-bench-2.0": 35.1
-      }
-    },
-    {
-      "model_id": "anthropic/claude-opus-4.5",
-      "name": "Claude Opus 4.5",
-      "developer": "Anthropic",
-      "scores": {
-        "terminal-bench-2.0": 63.1
-      }
-    },
-    {
-      "model_id": "anthropic/claude-opus-4.6",
-      "name": "Claude Opus 4.6",
-      "developer": "Anthropic",
-      "scores": {
-        "terminal-bench-2.0": 74.7
-      }
-    },
-    {
-      "model_id": "anthropic/claude-sonnet-4.5",
-      "name": "Claude Sonnet 4.5",
-      "developer": "Anthropic",
-      "scores": {
-        "terminal-bench-2.0": 42.5
-      }
-    },
-    {
-      "model_id": "deepseek/deepseek-v3.2",
-      "name": "DeepSeek-V3.2",
-      "developer": "DeepSeek",
-      "scores": {
-        "terminal-bench-2.0": 39.6
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-flash",
-      "name": "Gemini 2.5 Flash",
-      "developer": "Google",
-      "scores": {
-        "terminal-bench-2.0": 16.9
-      }
-    },
-    {
-      "model_id": "google/gemini-2.5-pro",
-      "name": "Gemini 2.5 Pro",
-      "developer": "Google",
-      "scores": {
-        "terminal-bench-2.0": 19.6
-      }
-    },
-    {
-      "model_id": "google/gemini-3-flash",
-      "name": "Gemini 3 Flash",
-      "developer": "Google",
-      "scores": {
-        "terminal-bench-2.0": 47.4
-      }
-    },
-    {
-      "model_id": "google/gemini-3-pro",
-      "name": "Gemini 3 Pro",
-      "developer": "Google",
-      "scores": {
-        "terminal-bench-2.0": 56.9
-      }
-    },
-    {
-      "model_id": "google/gemini-3.1-pro",
-      "name": "Gemini 3.1 Pro",
-      "developer": "Google",
-      "scores": {
-        "terminal-bench-2.0": 74.8
-      }
-    },
-    {
-      "model_id": "minimax/minimax-m2",
-      "name": "MiniMax M2",
-      "developer": "MiniMax",
-      "scores": {
-        "terminal-bench-2.0": 30.0
-      }
-    },
-    {
-      "model_id": "minimax/minimax-m2.1",
-      "name": "MiniMax M2.1",
-      "developer": "MiniMax",
-      "scores": {
-        "terminal-bench-2.0": 29.2
-      }
-    },
-    {
-      "model_id": "minimax/minimax-m2.5",
-      "name": "Minimax m2.5",
-      "developer": "Minimax",
-      "scores": {
-        "terminal-bench-2.0": 42.2
-      }
-    },
-    {
-      "model_id": "moonshot-ai/kimi-k2-instruct",
-      "name": "Kimi K2 Instruct",
-      "developer": "Moonshot AI",
-      "scores": {
-        "terminal-bench-2.0": 27.8
-      }
-    },
-    {
-      "model_id": "moonshot-ai/kimi-k2-thinking",
-      "name": "Kimi K2 Thinking",
-      "developer": "Moonshot AI",
-      "scores": {
-        "terminal-bench-2.0": 35.7
-      }
-    },
-    {
-      "model_id": "moonshot-ai/kimi-k2.5",
-      "name": "Kimi K2.5",
-      "developer": "Kimi",
-      "scores": {
-        "terminal-bench-2.0": 43.2
-      }
-    },
-    {
-      "model_id": "multiple/multiple",
-      "name": "Multiple",
-      "developer": "Multiple",
-      "scores": {
-        "terminal-bench-2.0": 59.1
-      }
-    },
-    {
-      "model_id": "openai/gpt-5",
-      "name": "GPT-5",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 49.6
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-codex",
-      "name": "GPT-5-Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 44.3
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-mini",
-      "name": "GPT-5-Mini",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 31.9
-      }
-    },
-    {
-      "model_id": "openai/gpt-5-nano",
-      "name": "GPT-5-Nano",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 7.0
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.1",
-      "name": "GPT-5.1",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 47.6
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.1-codex",
-      "name": "GPT-5.1-Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 53.5
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.1-codex-max",
-      "name": "GPT-5.1-Codex-Max",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 60.4
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.1-codex-mini",
-      "name": "GPT-5.1-Codex-Mini",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 43.1
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2",
-      "name": "GPT-5.2",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 64.9
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.2-codex",
-      "name": "GPT-5.2-Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 66.5
-      }
-    },
-    {
-      "model_id": "openai/gpt-5.3-codex",
-      "name": "GPT-5.3-Codex",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 74.6
-      }
-    },
-    {
-      "model_id": "openai/gpt-oss-120b",
-      "name": "GPT-OSS-120B",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 18.7
-      }
-    },
-    {
-      "model_id": "openai/gpt-oss-20b",
-      "name": "GPT-OSS-20B",
-      "developer": "OpenAI",
-      "scores": {
-        "terminal-bench-2.0": 3.4
-      }
-    },
-    {
-      "model_id": "xai/grok-4",
-      "name": "Grok 4",
-      "developer": "xAI",
-      "scores": {
-        "terminal-bench-2.0": 23.1
-      }
-    },
-    {
-      "model_id": "xai/grok-code-fast-1",
-      "name": "Grok Code Fast 1",
-      "developer": "xAI",
-      "scores": {
-        "terminal-bench-2.0": 14.2
-      }
-    },
-    {
-      "model_id": "zhipu-ai/glm-4.6",
-      "name": "GLM 4.6",
-      "developer": "Z.ai",
-      "scores": {
-        "terminal-bench-2.0": 24.5
-      }
-    },
-    {
-      "model_id": "zhipu-ai/glm-4.7",
-      "name": "GLM 4.7",
-      "developer": "Z-AI",
-      "scores": {
-        "terminal-bench-2.0": 33.3
-      }
-    },
-    {
-      "model_id": "zhipu-ai/glm-5",
-      "name": "GLM 5",
-      "developer": "Z-AI",
-      "scores": {
-        "terminal-bench-2.0": 52.4
-      }
-    }
-  ]
-}

data/benchmarks/theory_of_mind.json

@@ -1,12 +0,0 @@
-{
-  "models": [
-    {
-      "model_id": "Qwen/Qwen2.5-3B-Instruct",
-      "name": "Qwen2.5-3B-Instruct",
-      "developer": "Qwen",
-      "scores": {
-        "accuracy on theory_of_mind for scorer model_graded_fact": 0.78
-      }
-    }
-  ]
-}

lib/benchmark-metadata.ts

@@ -1,98 +1,66 @@
 import "server-only"
 
-import { promises as fs, type Dirent } from "fs"
-import path from "path"
-
 import type { BenchmarkCard } from "@/lib/benchmark-schema"
-import { normalizeBenchmarkKey, candidateBenchmarkKeys as candidateKeys } from "@/lib/benchmark-metadata-utils"
+import { candidateBenchmarkKeys as candidateKeys, normalizeBenchmarkKey } from "@/lib/benchmark-metadata-utils"
+import { fetchBenchmarkMetadataMap } from "@/lib/hf-data"
 
 export { normalizeBenchmarkKey }
 
-interface IndexedBenchmarkDetailFile {
-  benchmark_cards?: Record<string, BenchmarkCard>
-}
-
-function getBenchmarkDataDirectory() {
-  return path.join(process.cwd(), "data", "benchmarks")
-}
+let cachedMapPromise: Promise<Map<string, BenchmarkCard>> | null = null
 
-async function readEmbeddedBenchmarkCards(): Promise<Map<string, BenchmarkCard>> {
-  const dir = getBenchmarkDataDirectory()
+async function readPipelineBenchmarkCards(): Promise<Map<string, BenchmarkCard>> {
+  const cards = await fetchBenchmarkMetadataMap()
   const map = new Map<string, BenchmarkCard>()
 
-  let entries: Dirent[]
-  try {
-    entries = await fs.readdir(dir, { withFileTypes: true })
-  } catch {
-    return map
-  }
+  for (const card of Object.values(cards)) {
+    if (!card?.benchmark_details?.name) {
+      continue
+    }
 
-  const jsonFiles = entries.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
-
-  await Promise.all(
-    jsonFiles.map(async (entry) => {
-      try {
-        const raw = await fs.readFile(path.join(dir, entry.name), "utf8")
-        const parsed = JSON.parse(raw) as IndexedBenchmarkDetailFile
-        const embeddedCards = parsed.benchmark_cards
-
-        if (!embeddedCards || typeof embeddedCards !== "object") {
-          return
-        }
-
-        for (const [metricName, card] of Object.entries(embeddedCards)) {
-          if (!card?.benchmark_details?.name) {
-            continue
-          }
-
-          for (const key of candidateKeys(metricName)) {
-            if (!map.has(key)) map.set(key, card)
-          }
-
-          for (const key of candidateKeys(card.benchmark_details.name)) {
-            if (!map.has(key)) map.set(key, card)
-          }
-        }
-      } catch (err) {
-        console.warn(`benchmark-metadata: failed to load embedded cards from ${entry.name}:`, err)
+    for (const key of candidateKeys(card.benchmark_details.name)) {
+      if (!map.has(key)) {
+        map.set(key, card)
       }
-    })
-  )
+    }
+  }
 
   return map
 }
 
-let cachedMapPromise: Promise<Map<string, BenchmarkCard>> | null = null
-
 function getMap(): Promise<Map<string, BenchmarkCard>> {
-  if (process.env.NODE_ENV === "production") {
-    if (!cachedMapPromise) cachedMapPromise = readEmbeddedBenchmarkCards()
-    return cachedMapPromise
+  if (!cachedMapPromise) {
+    cachedMapPromise = readPipelineBenchmarkCards()
   }
-  return readEmbeddedBenchmarkCards()
+
+  return cachedMapPromise
 }
 
-/** Look up a BenchmarkCard by any commonly-used benchmark name. Returns null if not found. */
 export async function getBenchmarkCard(benchmarkName: string): Promise<BenchmarkCard | null> {
   const map = await getMap()
+
   for (const key of candidateKeys(benchmarkName)) {
     const card = map.get(key)
-    if (card) return card
+    if (card) {
+      return card
+    }
   }
+
   return null
 }
 
-/** Returns all loaded BenchmarkCards keyed by their normalised canonical name. */
 export async function getAllBenchmarkCards(): Promise<Record<string, BenchmarkCard>> {
   const map = await getMap()
-  // Deduplicate: only emit one entry per card (by canonical name)
   const seen = new Set<BenchmarkCard>()
   const result: Record<string, BenchmarkCard> = {}
-  for (const [key, card] of map) {
-    if (!seen.has(card)) {
-      seen.add(card)
-      result[normalizeBenchmarkKey(card.benchmark_details.name)] = card
+
+  for (const card of map.values()) {
+    if (seen.has(card)) {
+      continue
     }
+
+    seen.add(card)
+    result[normalizeBenchmarkKey(card.benchmark_details.name)] = card
   }
+
   return result
 }

lib/model-data.ts

@@ -1039,6 +1039,179 @@ function aggregateBenchmarkSummaries(
   }
 }
 
+const SYNTHETIC_MATRIX_EVAL_PREFIX = "matrix__"
+
+function buildSingleMetricSuiteMatrixSummary(
+  details: HFEvalDetail[],
+  suiteKey: string
+): BenchmarkEvalSummary | null {
+  if (details.length < 2) {
+    return null
+  }
+
+  const suiteDisplayName = getBenchmarkDisplayName(suiteKey)
+  const validDetails = [...details]
+    .filter((detail) => (detail.metrics?.length ?? 0) === 1 && extractDetailSubtasks(detail).length === 0)
+    .sort((left, right) =>
+      (left.benchmark_leaf_name || left.eval_summary_id).localeCompare(right.benchmark_leaf_name || right.eval_summary_id)
+    )
+
+  if (validDetails.length < 2) {
+    return null
+  }
+
+  const leaderboardMetrics: NonNullable<BenchmarkEvalSummary["leaderboard_metrics"]> = []
+  const rowStates = new Map<
+    string,
+    NonNullable<BenchmarkEvalSummary["leaderboard_rows"]>[number] & { _timestampValue: number }
+  >()
+
+  let metricConfig: BenchmarkEvalSummary["metric_config"] | null = null
+  let benchmarkCard: BenchmarkCard | undefined
+  const metricNames = new Set<string>()
+
+  for (const detail of validDetails) {
+    const metric = detail.metrics?.[0]
+    if (!metric) {
+      continue
+    }
+
+    if (!metricConfig) {
+      metricConfig = toSummaryMetricConfig(metric)
+    }
+
+    if (!benchmarkCard && detail.benchmark_card) {
+      benchmarkCard = detail.benchmark_card
+    }
+
+    const summaryMetric = toBenchmarkSummaryMetric(metric)
+    metricNames.add(summaryMetric.metric_name)
+    const subtaskKey = detail.benchmark_leaf_key || slugifyEvalId(detail.eval_summary_id)
+    const subtaskName = detail.benchmark_leaf_name || detail.canonical_display_name || detail.eval_summary_id || subtaskKey
+    const metricToken =
+      summaryMetric.metric_summary_id ||
+      summaryMetric.metric_key ||
+      slugifyEvalId(summaryMetric.display_name)
+    const columnKey = ["subtask", subtaskKey, metricToken].join(":")
+
+    leaderboardMetrics.push({
+      column_key: columnKey,
+      metric_summary_id: summaryMetric.metric_summary_id,
+      metric_name: summaryMetric.metric_name,
+      display_name: summaryMetric.display_name,
+      canonical_display_name: summaryMetric.canonical_display_name,
+      lower_is_better: summaryMetric.lower_is_better,
+      unit: summaryMetric.unit,
+      scope: "subtask",
+      subtask_key: subtaskKey,
+      subtask_name: subtaskName,
+    })
+
+    const benchmarkKey = detail.benchmark ?? suiteKey
+    const sourceName = detail.source_data?.dataset_name || benchmarkKey
+    const sourceOrganization = detail.source_data?.hf_repo || sourceName
+    const sourceMetadata: SourceMetadata = {
+      source_type: "documentation",
+      source_name: sourceName,
+      source_organization_name: sourceOrganization,
+      evaluator_relationship: "other",
+    }
+    const sourceData = detail.source_data ?? { dataset_name: benchmarkKey }
+
+    for (const modelResult of metric.model_results ?? []) {
+      const modelId = modelResult.model_id || modelResult.model_name
+      if (!modelId) {
+        continue
+      }
+
+      const nextTimestamp = normalizeEvalTimestamp(modelResult.retrieved_timestamp ?? "")
+      const existing = rowStates.get(modelId)
+
+      if (!existing) {
+        rowStates.set(modelId, {
+          model_info: {
+            name: modelResult.model_name ?? "",
+            id: modelId,
+            developer: modelResult.developer ?? "",
+          },
+          model_route_id: modelResult.model_route_id,
+          evaluation_timestamp: modelResult.retrieved_timestamp ?? "",
+          source_metadata: sourceMetadata,
+          source_data: sourceData,
+          values: { [columnKey]: modelResult.score ?? null },
+          metrics_present: 0,
+          _timestampValue: nextTimestamp,
+        })
+        continue
+      }
+
+      existing.values[columnKey] = modelResult.score ?? null
+      if (!existing.model_route_id && modelResult.model_route_id) {
+        existing.model_route_id = modelResult.model_route_id
+      }
+      if (nextTimestamp >= existing._timestampValue) {
+        existing.evaluation_timestamp = modelResult.retrieved_timestamp ?? existing.evaluation_timestamp
+        existing.source_metadata = sourceMetadata
+        existing.source_data = sourceData
+        existing._timestampValue = nextTimestamp
+      }
+    }
+  }
+
+  if (leaderboardMetrics.length < 2) {
+    return null
+  }
+
+  const sharedMetricName = metricNames.size === 1 ? Array.from(metricNames)[0] : undefined
+  const suiteMetricConfig = metricConfig
+    ? {
+        ...metricConfig,
+        evaluation_description: sharedMetricName ?? metricConfig.evaluation_description,
+      }
+    : {
+        evaluation_description: sharedMetricName ?? "",
+        lower_is_better: false,
+        score_type: "continuous" as const,
+        min_score: 0,
+        max_score: 1,
+      }
+
+  const leaderboardRows = Array.from(rowStates.values()).map(({ _timestampValue, ...row }) => ({
+    ...row,
+    metrics_present: leaderboardMetrics.reduce(
+      (count, metric) => count + (typeof row.values[metric.column_key] === "number" ? 1 : 0),
+      0
+    ),
+  }))
+
+  return {
+    evaluation_name: suiteDisplayName,
+    evaluation_id: `${SYNTHETIC_MATRIX_EVAL_PREFIX}${suiteKey}`,
+    canonical_display_name: suiteDisplayName,
+    composite_benchmark_key: suiteKey,
+    composite_benchmark_name: suiteDisplayName,
+    category: inferCategoryFromBenchmark(suiteDisplayName),
+    metric_config: suiteMetricConfig,
+    model_results: [],
+    models_count: leaderboardRows.length,
+    evaluator_names: [],
+    source_types: [],
+    latest_source_name: suiteDisplayName,
+    third_party_ratio: 0,
+    missing_generation_config_count: 0,
+    best_model: null,
+    worst_model: null,
+    avg_score: 0,
+    avg_score_norm: 0,
+    benchmark_card: benchmarkCard,
+    metrics_count: leaderboardMetrics.length,
+    metric_names: leaderboardMetrics.map((metric) => `${metric.subtask_name} / ${metric.metric_name}`),
+    source_data: { dataset_name: suiteDisplayName },
+    leaderboard_metrics: leaderboardMetrics,
+    leaderboard_rows: leaderboardRows,
+  }
+}
+
 // ---------------------------------------------------------------------------
 // Public API
 // ---------------------------------------------------------------------------
@@ -1361,6 +1534,34 @@ export async function getEvalSummaryById(evalId: string) {
     return aggregateBenchmarkSummaries(validSummaries, aggregateKey)
   }
 
+  if (evalId.startsWith(SYNTHETIC_MATRIX_EVAL_PREFIX)) {
+    const suiteKey = evalId.replace(new RegExp(`^${SYNTHETIC_MATRIX_EVAL_PREFIX}`), "")
+    const normalizedSuiteKey = normalizeBenchmarkKeyForLookup(suiteKey)
+    const { evals } = await fetchHFEvalListLite()
+    const matchingEvals = evals.filter((entry) => {
+      if (entry.is_summary_score) {
+        return false
+      }
+
+      const parentKey = normalizeBenchmarkKeyForLookup(
+        entry.benchmark_parent_key || entry.benchmark_family_key || entry.benchmark
+      )
+      return parentKey === normalizedSuiteKey
+    })
+
+    if (matchingEvals.length < 2) {
+      return null
+    }
+
+    const details = await Promise.all(
+      matchingEvals.map(async (entry) => fetchHFEvalDetail(entry.eval_summary_id))
+    )
+
+    const validDetails = details.filter((detail): detail is HFEvalDetail => detail !== null)
+    const syntheticSummary = buildSingleMetricSuiteMatrixSummary(validDetails, suiteKey)
+    return syntheticSummary ? attachBenchmarkCardToSummary(syntheticSummary) : null
+  }
+
   // Direct eval lookup
   const detail = await fetchHFEvalDetail(evalId)
   if (detail) {