elizaos--eliza/packages/benchmarks/registry/commands.py

from __future__ import annotations

import os
import sys
from pathlib import Path
from typing import Mapping, cast

try:
    from benchmarks.bench_cli_types import (
        BenchmarkDefinition,
        BenchmarkRequirements,
        JSONValue,
        ModelSpec,
        find_latest_file,
        load_json_file,
    )
    from benchmarks.registry.scores import (
        _score_from_abliteration_robustness_json,
        _score_from_action_calling_json,
        _score_from_agentbench_json,
        _score_from_bfcl_json,
        _score_from_clawbench_json,
        _score_from_configbench_json,
        _score_from_contextbench_json,
        _score_from_eliza_format_json,
        _score_from_gauntlet_json,
        _score_from_gsm8k_json,
        _score_from_hermes_env_json,
        _score_from_humaneval_json,
        _score_from_hyperliquid_bench_json,
        _score_from_lifeops_bench_json,
        _score_from_meeting_transcription_proof_json,
        _score_from_mind2web_json,
        _score_from_mint_json,
        _score_from_multitask_bench_json,
        _score_from_mmau_json,
        _score_from_mmlu_json,
        _score_from_mt_bench_json,
        _score_from_openclaw_bench_json,
        _score_from_orchestrator_lifecycle_json,
        _score_from_osworld_json,
        _score_from_realm_json,
        _score_from_recall_json,
        _score_from_rlmbench_json,
        _score_from_scambench_json,
        _score_from_social_alpha_json,
        _score_from_solana_json,
        _score_from_swebench_json,
        _score_from_swebench_orchestrated_json,
        _score_from_taubench_json,
        _score_from_terminalbench_json,
        _score_from_trajectory_replay_json,
        _score_from_trust_json,
        _score_from_vendingbench_json,
        _score_from_visualwebbench_json,
        _score_from_vision_language_json,
        _score_from_voiceagentbench_json,
        _score_from_voicebench_json,
        _score_from_voicebench_quality_json,
        _score_from_webshop_json,
        _score_from_woobench_json,
    )
except ImportError:
    from bench_cli_types import (  # type: ignore[no-redef]
        BenchmarkDefinition,
        BenchmarkRequirements,
        JSONValue,
        ModelSpec,
        find_latest_file,
        load_json_file,
    )
    from registry.scores import (  # type: ignore[no-redef]
        _score_from_abliteration_robustness_json,
        _score_from_action_calling_json,
        _score_from_agentbench_json,
        _score_from_bfcl_json,
        _score_from_clawbench_json,
        _score_from_configbench_json,
        _score_from_contextbench_json,
        _score_from_eliza_format_json,
        _score_from_gauntlet_json,
        _score_from_gsm8k_json,
        _score_from_hermes_env_json,
        _score_from_humaneval_json,
        _score_from_hyperliquid_bench_json,
        _score_from_lifeops_bench_json,
        _score_from_meeting_transcription_proof_json,
        _score_from_mind2web_json,
        _score_from_mint_json,
        _score_from_multitask_bench_json,
        _score_from_mmau_json,
        _score_from_mmlu_json,
        _score_from_mt_bench_json,
        _score_from_openclaw_bench_json,
        _score_from_orchestrator_lifecycle_json,
        _score_from_osworld_json,
        _score_from_realm_json,
        _score_from_recall_json,
        _score_from_rlmbench_json,
        _score_from_scambench_json,
        _score_from_social_alpha_json,
        _score_from_solana_json,
        _score_from_swebench_json,
        _score_from_swebench_orchestrated_json,
        _score_from_taubench_json,
        _score_from_terminalbench_json,
        _score_from_trajectory_replay_json,
        _score_from_trust_json,
        _score_from_vendingbench_json,
        _score_from_visualwebbench_json,
        _score_from_vision_language_json,
        _score_from_voiceagentbench_json,
        _score_from_voicebench_json,
        _score_from_voicebench_quality_json,
        _score_from_webshop_json,
        _score_from_woobench_json,
    )


_MINT_CATEGORY_TO_SUBTASKS: dict[str, tuple[str, ...]] = {
    "reasoning": ("gsm8k",),
    "code_generation": ("humaneval",),
    "coding": ("humaneval",),
    "decision_making": (),
}


def _validate_osworld_dry_run_label(extra: Mapping[str, JSONValue]) -> None:
    agent_label = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
    mode_label = (
        str(extra.get("run_mode") or extra.get("mode") or extra.get("suite") or "")
        .strip()
        .lower()
    )
    marked_smoke = extra.get("smoke") is True or mode_label in {
        "smoke",
        "dry_run",
        "dry-run",
        "smoke_dry_run",
    }
    if agent_label in {"eliza", "hermes", "openclaw"} and not marked_smoke:
        raise ValueError(
            "osworld dry_run is smoke-only. Set smoke=true or run_mode=smoke "
            "for smoke rows; omit dry_run for real VM benchmark rows."
        )


def _mint_subtasks_from_extra(extra: Mapping[str, JSONValue]) -> list[str]:
    subtasks = extra.get("subtasks")
    if isinstance(subtasks, list) and all(isinstance(x, str) for x in subtasks):
        return [str(item) for item in subtasks if str(item).strip()]

    categories = extra.get("categories")
    if not isinstance(categories, list) or not all(isinstance(x, str) for x in categories):
        return []

    selected: list[str] = []
    for category in categories:
        mapped = _MINT_CATEGORY_TO_SUBTASKS.get(category.strip().lower())
        if mapped is None:
            selected.append(category)
            continue
        selected.extend(mapped)

    deduped: list[str] = []
    for item in selected:
        normalized = item.strip()
        if normalized and normalized not in deduped:
            deduped.append(normalized)
    return deduped


def _expand_scenarios_requested(extra: Mapping[str, JSONValue]) -> bool:
    if extra.get("expand_scenarios") is True or extra.get("include_edge_scenarios") is True:
        return True
    return os.environ.get("EXPAND_SCENARIOS", "").strip().lower() in {
        "1",
        "true",
        "yes",
        "on",
    } or os.environ.get("INCLUDE_EDGE_SCENARIOS", "").strip().lower() in {
        "1",
        "true",
        "yes",
        "on",
    }


def _flag_requested(extra: Mapping[str, JSONValue], *keys: str) -> bool:
    if any(extra.get(key) is True for key in keys):
        return True
    truthy = {"1", "true", "yes", "on"}
    return any(os.environ.get(key.upper(), "").strip().lower() in truthy for key in keys)


def _append_scenario_control_flags(args: list[str], extra: Mapping[str, JSONValue]) -> None:
    if _expand_scenarios_requested(extra):
        args.append("--expand-scenarios")
    if _flag_requested(extra, "count_scenarios"):
        args.append("--count-scenarios")
    if _flag_requested(extra, "validate_scenarios"):
        args.append("--validate-scenarios")


def get_benchmark_registry(repo_root: Path) -> list[BenchmarkDefinition]:
    python = sys.executable

    def repo(path: str) -> str:
        return str((repo_root / path).resolve())

    def _bfcl_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [python, "-m", "benchmarks.bfcl", "run", "--output", str(output_dir)]
        provider_name = (model.provider or "").strip().lower()
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        # Route LLM-backed providers through the eliza TS bridge so the
        # ElizaBFCLAgent + registered runtime is exercised.
        bridge_providers = {"cerebras", "openai", "groq", "openrouter", "vllm", "eliza"}
        if agent in {"eliza", "hermes", "openclaw"}:
            args.extend(["--provider", agent])
            if model.model:
                args.extend(["--model", model.model])
        elif provider_name in bridge_providers:
            args.extend(["--provider", "eliza"])
            if model.model:
                args.extend(["--model", model.model])
        else:
            if model.provider:
                args.extend(["--provider", model.provider])
            if model.model:
                model_name = model.model
                if provider_name == "groq" and not model_name.startswith("groq/"):
                    model_name = f"groq/{model_name}"
                args.extend(["--model", model_name])
        if extra.get("mock") is True or provider_name == "mock":
            args.append("--mock")
        sample = extra.get("sample", extra.get("max_tasks"))
        if isinstance(sample, int) and sample > 0:
            args.extend(["--sample", str(sample)])
        seed = extra.get("seed")
        if isinstance(seed, int):
            args.extend(["--seed", str(seed)])
        elif isinstance(sample, int) and sample > 0:
            args.extend(["--seed", "0"])
        max_per_category = extra.get("max_per_category")
        if isinstance(max_per_category, int) and max_per_category > 0:
            args.extend(["--max-per-category", str(max_per_category)])
        categories = extra.get("categories")
        if isinstance(categories, list) and all(isinstance(x, str) for x in categories):
            args.extend(["--categories", ",".join(cast(list[str], categories))])
        local_data = extra.get("local_data")
        if isinstance(local_data, str) and local_data.strip():
            args.extend(["--local-data", local_data])
        if extra.get("no_exec") is True:
            args.append("--no-exec")
        _append_scenario_control_flags(args, extra)
        return args

    def _bfcl_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="bfcl_results_*.json")

    def _realm_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [python, "-m", "benchmarks.realm.cli", "--output", str(output_dir)]
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        provider_name = (model.provider or "").strip().lower()
        data_path = extra.get("data_path")
        if isinstance(data_path, str) and data_path.strip():
            args.extend(["--data-path", data_path.strip()])
        categories = extra.get("categories")
        if isinstance(categories, list) and all(isinstance(x, str) for x in categories):
            args.extend(["--categories", *cast(list[str], categories)])
        elif extra.get("use_sample_tasks") is True or extra.get("mock") is True or provider_name == "mock":
            args.append("--use-sample-tasks")
        execution_model = extra.get("execution_model")
        if isinstance(execution_model, str) and execution_model.strip():
            args.extend(["--execution-model", execution_model.strip()])
        elif extra.get("max_tasks") == 1:
            args.extend(["--execution-model", "sequential"])
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        max_steps = extra.get("max_steps")
        if isinstance(max_steps, int) and max_steps > 0:
            args.extend(["--max-steps", str(max_steps)])
        timeout = extra.get("timeout")
        if isinstance(timeout, int) and timeout > 0:
            args.extend(["--timeout", str(timeout)])
        if model.model:
            # REALM treats --model as a reporting label only; the actual LLM is
            # picked by the in-process elizaOS Python runtime (default) or by the
            # TS bridge (when --provider eliza is set).
            args.extend(["--model", model.model])
        if extra.get("mock") is True or provider_name == "mock":
            args.append("--mock")
        # Route the planning loop through the TS benchmark server when the
        # caller asks for the eliza agent or any LLM-backed provider.
        if agent in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--provider", agent])
        elif provider_name in {
            "eliza",
            "cerebras",
            "openai",
            "groq",
            "openrouter",
            "vllm",
        }:
            args.extend(["--provider", "eliza"])
        _append_scenario_control_flags(args, extra)
        return args

    def _realm_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="realm-benchmark-*.json")

    def _mint_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [python, repo("benchmarks/mint/run_benchmark.py"), "--output-dir", str(output_dir)]
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        subtasks = _mint_subtasks_from_extra(extra)
        if subtasks:
            args.extend(["--subtasks", *subtasks])
        # Route LLM calls through the elizaOS TS benchmark server when the caller
        # asks for the eliza agent; otherwise forward real provider/model labels
        # to the direct OpenAI-compatible runtime instead of silently using mock.
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        provider_name = (model.provider or "").strip().lower()
        if agent in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--provider", agent])
            if model.model:
                args.extend(["--model", model.model])
            base_url = extra.get("base_url")
            if isinstance(base_url, str) and base_url.strip():
                args.extend(["--base-url", base_url.strip()])
        elif provider_name == "eliza":
            args.extend(["--provider", "eliza"])
        elif provider_name in {
            "cerebras",
            "openai",
            "groq",
            "openrouter",
            "vllm",
        }:
            args.extend(["--provider", provider_name])
            if model.model:
                args.extend(["--model", model.model])
            base_url = extra.get("base_url")
            if isinstance(base_url, str) and base_url.strip():
                args.extend(["--base-url", base_url.strip()])
        elif provider_name in {
            "eliza",
        }:
            args.extend(["--provider", "eliza"])
        elif provider_name in {"mock", ""}:
            args.extend(["--provider", "mock"])
        elif model.provider:
            raise ValueError(f"mint: unsupported provider '{model.provider}'")
        if extra.get("no_ablation") is True:
            args.append("--no-ablation")
        if extra.get("no_tools") is True:
            args.append("--no-tools")
        if extra.get("no_feedback") is True:
            args.append("--no-feedback")
        if extra.get("no_docker") is True:
            args.append("--no-docker")
        if extra.get("no_report") is True:
            args.append("--no-report")
        max_turns = extra.get("max_turns")
        if isinstance(max_turns, int) and max_turns > 0:
            args.extend(["--max-turns", str(max_turns)])
        timeout = extra.get("timeout")
        if isinstance(timeout, int) and timeout > 0:
            args.extend(["--timeout", str(timeout)])
        return args

    def _mint_result(output_dir: Path) -> Path:
        return output_dir / "mint-benchmark-results.json"

    def _agentbench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [
            python,
            "-m",
            "elizaos_agentbench.cli",
            "run",
            "--output",
            str(output_dir),
        ]
        envs = extra.get("env")
        if isinstance(envs, list) and all(isinstance(x, str) for x in envs):
            env_aliases = {
                "db": "database",
                "kg": "kg",
                "lt": "lateral",
                "os": "os",
                "ws": "webshop",
                "all": "all",
            }
            mapped_envs = [env_aliases.get(env, env) for env in cast(list[str], envs)]
            for env in mapped_envs:
                args.extend(["--env", env])
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        if extra.get("no_docker") is True:
            args.append("--no-docker")
        # Agent runtime selection
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        if agent in {"hermes", "openclaw", "smithers"}:
            args.extend(["--runtime", agent])
        elif agent == "eliza" or extra.get("elizaos") is True:
            args.extend(["--runtime", "bridge"])
        else:
            args.extend(["--runtime", "mock"])
        _append_scenario_control_flags(args, extra)
        _ = model
        return args

    def _agentbench_result(output_dir: Path) -> Path:
        return output_dir / "agentbench-results.json"

    def _contextbench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        agent = extra.get("agent")
        if agent == "eliza":
            provider_str = "eliza"
        else:
            provider = extra.get("provider")
            provider_name = ""
            if isinstance(provider, str):
                provider_name = provider.strip().lower()
            elif model.provider:
                provider_name = model.provider.strip().lower()
            # Route all OpenAI-compatible LLM providers (including cerebras)
            # through the eliza TS bridge instead of falling back to mock.
            provider_map: dict[str, str] = {
                "openai": "eliza",
                "groq": "eliza",
                "openrouter": "eliza",
                "vllm": "eliza",
                "cerebras": "eliza",
                "eliza": "eliza",
                "anthropic": "anthropic",
            }
            provider_str = provider_map.get(provider_name, "mock")
        args = [
            python,
            repo("benchmarks/context-bench/run_benchmark.py"),
            "--provider",
            provider_str,
            "--output-dir",
            str(output_dir),
        ]
        harness = str(extra.get("agent") or extra.get("harness") or "eliza").strip().lower()
        if harness in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--harness", harness])
        quick = extra.get("quick")
        if quick is True:
            args.append("--quick")
        context_lengths = extra.get("context_lengths")
        if isinstance(context_lengths, list) and all(isinstance(x, int) for x in context_lengths):
            args.extend(["--context-lengths", ",".join(str(x) for x in cast(list[int], context_lengths))])
        elif isinstance(context_lengths, str) and context_lengths.strip():
            args.extend(["--context-lengths", context_lengths.strip()])
        positions = extra.get("positions")
        if isinstance(positions, list) and all(isinstance(x, str) for x in positions):
            args.extend(["--positions", ",".join(cast(list[str], positions))])
        elif isinstance(positions, str) and positions.strip():
            args.extend(["--positions", positions.strip()])
        tasks_per_position = extra.get("tasks_per_position")
        if isinstance(tasks_per_position, int) and tasks_per_position > 0:
            args.extend(["--tasks-per-position", str(tasks_per_position)])
        _ = model
        return args

    def _contextbench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="context_bench_*.json")

    def _recall_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        # recall-bench (#9956): drives the REAL @elizaos/core recall path over a
        # labeled, document-scale corpus. No model/provider — the embedding is a
        # deterministic in-bench function, so it is secret-free and CI-safe.
        tier = str(extra.get("tier") or "1k").strip().lower()
        if tier not in {"smoke", "1k", "10k"}:
            tier = "1k"
        _ = model
        # `--conditions=eliza-source` resolves @elizaos/* to workspace source.
        return [
            "bun",
            "--conditions=eliza-source",
            "run.ts",
            "--tier",
            tier,
            "--out",
            str(output_dir),
        ]

    def _recall_result(output_dir: Path) -> Path:
        return output_dir / "recall-bench-results.json"

    def _terminalbench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        # Run module from its python project root.
        args = [
            python,
            "-m",
            "elizaos_terminal_bench.cli",
            "--output-dir",
            str(output_dir),
        ]
        provider_name = (model.provider or "").strip().lower()
        agent = extra.get("agent")
        # LLM-backed providers route through the eliza TS bridge so the
        # registered eliza agent + plugins are exercised. Hermes/OpenClaw also
        # use that Python bridge surface, but their delegate clients must keep
        # the real provider/model from the orchestrator environment.
        bridge_providers = {"cerebras", "openai", "groq", "openrouter", "vllm", "eliza"}
        if agent in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--agent-harness", str(agent)])
            if model.model:
                args.extend(["--model", model.model])
        elif provider_name in bridge_providers:
            if model.model:
                args.extend(["--model", model.model])
            args.extend(["--model-provider", "eliza"])
        else:
            if model.model:
                model_name = model.model
                if model.provider and "/" not in model_name:
                    model_name = f"{model.provider}/{model_name}"
                args.extend(["--model", model_name])
            if model.provider:
                args.extend(["--model-provider", model.provider])
        if model.temperature is not None:
            args.extend(["--temperature", str(model.temperature)])
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        task_ids = extra.get("task_ids")
        if isinstance(task_ids, list) and all(isinstance(x, str) for x in task_ids):
            args.extend(["--task-ids", *cast(list[str], task_ids)])
        elif isinstance(task_ids, str) and task_ids.strip():
            args.extend(["--task-ids", task_ids.strip()])
        single = extra.get("single")
        if isinstance(single, str) and single.strip():
            args.extend(["--single", single.strip()])
        timeout = extra.get("timeout")
        if isinstance(timeout, int) and timeout > 0:
            args.extend(["--timeout", str(timeout)])
        network_mode = extra.get("network_mode")
        if isinstance(network_mode, str) and network_mode.strip():
            args.extend(["--network-mode", network_mode.strip()])
        sample = extra.get("sample")
        if sample is True:
            args.append("--sample")
        _append_scenario_control_flags(args, extra)
        if extra.get("dry_run") is True:
            args.append("--dry-run")
        elif extra.get("no_docker") is True:
            args.append("--local-sandbox")
        if extra.get("oracle") is True:
            args.append("--oracle")
        if extra.get("no_markdown") is True:
            args.append("--no-markdown")
        if extra.get("no_sessions") is True:
            args.append("--no-sessions")
        if extra.get("no_leaderboard") is True:
            args.append("--no-leaderboard")
        return args

    def _terminalbench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="terminal-bench-*.json")

    def _tau_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [
            python,
            "-m",
            "elizaos_tau_bench.cli",
            "--output-dir",
            str(output_dir),
        ]
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        provider_name = (model.provider or "").strip().lower()
        if agent in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--agent-harness", agent])
            args.extend(["--agent-provider", model.provider or "cerebras"])
            if model.model:
                args.extend(["--agent-model", model.model])
        elif extra.get("mock") is True or provider_name == "mock":
            args.append("--mock")
        else:
            args.extend(["--agent-harness", "litellm"])
            if model.provider:
                args.extend(["--agent-provider", model.provider])
            if model.model:
                args.extend(["--agent-model", model.model])
        if model.temperature is not None:
            args.extend(["--agent-temperature", str(model.temperature)])
        _append_scenario_control_flags(args, extra)
        agent_max_turns = extra.get("agent_max_turns")
        if isinstance(agent_max_turns, int) and agent_max_turns > 0:
            args.extend(["--agent-max-turns", str(agent_max_turns)])
        if not (extra.get("mock") is True or provider_name == "mock"):
            user_provider = extra.get("user_provider")
            if not isinstance(user_provider, str) or not user_provider.strip():
                user_provider = model.provider or "openai"
            user_model = extra.get("user_model")
            if not isinstance(user_model, str) or not user_model.strip():
                user_model = model.model or "gpt-4o"
            args.extend(["--user-provider", user_provider.strip()])
            args.extend(["--user-model", user_model.strip()])

            if extra.get("no_llm_judge") is not True:
                judge_provider = extra.get("judge_provider")
                if not isinstance(judge_provider, str) or not judge_provider.strip():
                    judge_provider = model.provider or "openai"
                judge_model = extra.get("judge_model")
                if not isinstance(judge_model, str) or not judge_model.strip():
                    judge_model = model.model or "gpt-4o-mini"
                args.extend(["--judge-provider", judge_provider.strip()])
                args.extend(["--judge-model", judge_model.strip()])
        user_strategy = extra.get("user_strategy")
        if isinstance(user_strategy, str) and user_strategy.strip():
            args.extend(["--user-strategy", user_strategy.strip()])
        num_trials = extra.get("num_trials")
        if isinstance(num_trials, int) and num_trials > 0:
            args.extend(["--num-trials", str(num_trials)])
        pass_k_values = extra.get("pass_k_values")
        if isinstance(pass_k_values, list) and all(isinstance(k, int) and k > 0 for k in pass_k_values):
            args.extend(["--pass-k-values", *[str(k) for k in pass_k_values]])
        task_ids = extra.get("task_ids")
        if isinstance(task_ids, list) and all(isinstance(t, int) and t >= 0 for t in task_ids):
            args.extend(["--task-ids", *[str(t) for t in task_ids]])
        start_index = extra.get("start_index")
        if isinstance(start_index, int) and start_index >= 0:
            args.extend(["--start-index", str(start_index)])
        end_index = extra.get("end_index")
        if isinstance(end_index, int):
            args.extend(["--end-index", str(end_index)])
        max_tasks = extra.get("max_tasks", extra.get("max_tasks_per_domain"))
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks-per-domain", str(max_tasks)])
        sample = extra.get("sample")
        if sample is True or extra.get("use_sample_tasks") is True:
            args.append("--use-sample-tasks")
        domain = extra.get("domain")
        if isinstance(domain, str) and domain in {"retail", "airline", "both"}:
            args.extend(["--domain", domain])
        if extra.get("no_llm_judge") is True:
            args.append("--no-llm-judge")
        return args

    def _tau_result(output_dir: Path) -> Path:
        return output_dir / "report.json"

    def _vending_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [python, "-m", "elizaos_vending_bench.cli", "run", "--output-dir", str(output_dir)]
        if model.model:
            args.extend(["--model", model.model])
        provider_name = (model.provider or "").strip().lower()
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        if extra.get("mock") is True or provider_name == "mock":
            args.extend(["--provider", "heuristic"])
        elif agent in {"eliza", "hermes", "openclaw"} or provider_name == "cerebras":
            args.extend(["--provider", "eliza"])
        elif model.provider in {"openai", "anthropic", "groq", "heuristic", "eliza", "vllm"}:
            args.extend(["--provider", model.provider])
        if model.temperature is not None:
            args.extend(["--temperature", str(model.temperature)])
        runs = extra.get("runs")
        if not isinstance(runs, int):
            runs = extra.get("num_runs")
        if isinstance(runs, int) and runs > 0:
            args.extend(["--runs", str(runs)])
        elif extra.get("max_tasks") == 1:
            args.extend(["--runs", "1"])
        days = extra.get("days")
        if not isinstance(days, int):
            days = extra.get("max_days_per_run")
        if isinstance(days, int) and days > 0:
            args.extend(["--days", str(max(days, 3))])
        elif extra.get("max_tasks") == 1:
            args.extend(["--days", "3"])
        args.append("--starter-inventory")
        args.extend(["--max-actions-per-day", "6"])
        _append_scenario_control_flags(args, extra)
        return args

    def _vending_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="vending-bench-results-*.json")

    def _swe_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [python, "-m", "benchmarks.swe_bench.cli", "--output", str(output_dir)]
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        if agent in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--harness", agent])
        if model.model:
            args.extend(["--model", model.model])
        if model.provider:
            args.extend(["--provider", model.provider])
        max_instances = extra.get("max_instances")
        if isinstance(max_instances, int) and max_instances > 0:
            args.extend(["--max-instances", str(max_instances)])
        variant = extra.get("variant")
        if isinstance(variant, str) and variant in ("lite", "verified", "full"):
            args.extend(["--variant", variant])
        no_docker = extra.get("no_docker")
        if no_docker is True:
            args.append("--no-docker")
        provider_lower = (model.provider or "").strip().lower()
        if extra.get("mock") is True or provider_lower == "mock":
            args.append("--mock")
        _append_scenario_control_flags(args, extra)
        return args

    def _swe_orchestrated_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        args = [
            python,
            "-m",
            "benchmarks.swe_bench.cli",
            "--orchestrated",
            "--output",
            str(output_dir),
        ]
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        if agent in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--harness", agent])
        if model.model:
            args.extend(["--model", model.model])
        if model.provider:
            args.extend(["--provider", model.provider])
        max_instances = extra.get("max_instances")
        if isinstance(max_instances, int) and max_instances > 0:
            args.extend(["--max-instances", str(max_instances)])
        variant = extra.get("variant")
        if isinstance(variant, str) and variant in {"lite", "verified", "full"}:
            args.extend(["--variant", variant])
        max_steps = extra.get("max_steps")
        if isinstance(max_steps, int) and max_steps > 0:
            args.extend(["--max-steps", str(max_steps)])
        if extra.get("no_docker") is True:
            args.append("--no-docker")
        provider_lower = (model.provider or "").strip().lower()
        if extra.get("mock") is True or provider_lower == "mock":
            args.append("--mock")
        _append_scenario_control_flags(args, extra)

        execution_mode = extra.get("execution_mode")
        if isinstance(execution_mode, str) and execution_mode in {
            "orchestrated",
            "direct_shell",
        }:
            args.extend(["--execution-mode", execution_mode])

        providers = extra.get("providers")
        legacy_defaults = ["claude-code", "swe-agent", "codex"]
        if (
            agent in {"eliza", "hermes", "openclaw"}
            and isinstance(providers, list)
            and [str(p) for p in providers] == legacy_defaults
        ):
            args.extend(["--providers", agent])
        elif isinstance(providers, list):
            provider_values = [str(p) for p in providers if str(p).strip()]
            if provider_values:
                args.extend(["--providers", *provider_values])
        elif agent in {"eliza", "hermes", "openclaw"}:
            args.extend(["--providers", agent])
        if extra.get("matrix") is True:
            args.append("--matrix")
        if extra.get("no_baseline") is True:
            args.append("--no-baseline")
        if extra.get("allow_task_fallback") is True:
            args.append("--allow-task-fallback")
        orchestrator_model = extra.get("orchestrator_model")
        if isinstance(orchestrator_model, str) and orchestrator_model.strip():
            args.extend(["--orchestrator-model", orchestrator_model.strip()])
        trace_dir = extra.get("trace_dir")
        if isinstance(trace_dir, str) and trace_dir.strip():
            args.extend(["--trace-dir", trace_dir.strip()])
        required_caps = extra.get("required_capabilities")
        if isinstance(required_caps, list) and required_caps:
            args.extend(["--required-capabilities", ",".join(str(c) for c in required_caps)])
        elif isinstance(required_caps, str) and required_caps.strip():
            args.extend(["--required-capabilities", required_caps.strip()])
        if extra.get("strict_capabilities") is True:
            args.append("--strict-capabilities")
        return args

    def _swe_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="swe-bench-*.json")

    def _swe_orchestrated_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="orchestrated-*.json")

    def _orchestrator_lifecycle_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        args = [
            python,
            "-m",
            "benchmarks.orchestrator_lifecycle.cli",
            "--output",
            str(output_dir),
        ]
        if model.model:
            args.extend(["--model", model.model])
        if model.provider:
            args.extend(["--provider", model.provider])
        max_scenarios = extra.get("max_scenarios")
        if isinstance(max_scenarios, int) and max_scenarios > 0:
            args.extend(["--max-scenarios", str(max_scenarios)])
        scenario_filter = extra.get("scenario_filter")
        if isinstance(scenario_filter, str) and scenario_filter.strip():
            args.extend(["--scenario-filter", scenario_filter.strip()])
        seed = extra.get("seed")
        if isinstance(seed, int):
            args.extend(["--seed", str(seed)])
        if extra.get("strict") is True:
            args.append("--strict")
        # Default to bridge mode (real eliza TS agent) for any LLM-backed
        # provider; explicit `--extra '{"mode":"simulate"}'` opts into the
        # deterministic simulator for offline smoke-testing only.
        provider_name = (model.provider or "").strip().lower()
        mode_override = extra.get("mode")
        if isinstance(mode_override, str) and mode_override.strip() in {
            "bridge",
            "simulate",
        }:
            args.extend(["--mode", mode_override.strip()])
        elif provider_name in {"cerebras", "openai", "groq", "openrouter", "vllm", "anthropic", "google", "eliza"}:
            args.extend(["--mode", "bridge"])
        else:
            args.extend(["--mode", "simulate"])
        _append_scenario_control_flags(args, extra)
        return args

    def _orchestrator_lifecycle_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="orchestrator-lifecycle-*.json")

    def _mind2web_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [python, "-m", "benchmarks.mind2web", "--output", str(output_dir)]
        agent = extra.get("agent")
        provider_name = (model.provider or "").strip().lower()
        # Route LLM-backed providers through the eliza TS bridge so the actual
        # registered eliza agent + plugins are exercised, not the python mock.
        if agent == "eliza" or provider_name in {
            "cerebras",
            "openai",
            "groq",
            "openrouter",
            "vllm",
            "eliza",
        }:
            args.extend(["--real-llm", "--provider", "eliza"])
            if model.model:
                args.extend(["--model", model.model])
        else:
            if model.provider and provider_name != "mock":
                args.extend(["--provider", model.provider])
            real_llm = extra.get("real_llm")
            if real_llm is True:
                args.append("--real-llm")
            if model.model:
                if provider_name == "groq":
                    args.extend(["--groq-small-model", model.model])
                    args.extend(["--groq-large-model", model.model])
                else:
                    args.extend(["--model", model.model])
        if model.temperature is not None:
            args.extend(["--temperature", str(model.temperature)])
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        max_steps = extra.get("max_steps")
        if isinstance(max_steps, int) and max_steps > 0:
            args.extend(["--max-steps", str(max_steps)])
        sample = extra.get("sample")
        if sample is True:
            args.append("--sample")
        mock = extra.get("mock")
        provider_name = (model.provider or "").strip().lower()
        if mock is True or provider_name == "mock":
            args.append("--mock")
        _append_scenario_control_flags(args, extra)
        return args

    def _mind2web_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="mind2web-results*.json")

    def _visualwebbench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [
            python,
            "-m",
            "benchmarks.visualwebbench",
            "--output",
            str(output_dir),
        ]
        agent = extra.get("agent")
        provider_name = str(extra.get("model_provider") or model.provider or "").strip().lower()
        local_eliza_providers = {"local-eliza", "local_eliza", "eliza-local", "eliza_local"}
        if extra.get("mock") is True or provider_name == "mock":
            args.append("--mock")
            if model.model:
                args.extend(["--model", model.model])
        elif provider_name in local_eliza_providers:
            # Local eliza-1 VLM via llama-mtmd-cli — no agent server / API.
            args.extend(["--provider", provider_name])
            tier = str(extra.get("tier") or model.model or "eliza-1-9b").strip() or "eliza-1-9b"
            args.extend(["--model", tier])
        elif agent == "eliza" or provider_name in {
            "cerebras",
            "openai",
            "groq",
            "openrouter",
            "vllm",
            "eliza",
        }:
            args.extend(["--provider", "eliza"])
            if model.model:
                args.extend(["--model", model.model])
        else:
            args.append("--dry-run")

        if model.temperature is not None:
            args.extend(["--temperature", str(model.temperature)])
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        task_types = extra.get("task_types")
        if isinstance(task_types, list) and all(isinstance(x, str) for x in task_types):
            args.extend(["--task-types", ",".join(cast(list[str], task_types))])
        elif isinstance(task_types, str) and task_types.strip():
            args.extend(["--task-types", task_types.strip()])
        fixture_path = extra.get("fixture_path")
        if isinstance(fixture_path, str) and fixture_path.strip():
            args.extend(["--fixture-path", fixture_path.strip()])
        if (
            extra.get("sample") is True
            or extra.get("use_sample_tasks") is True
            or extra.get("mock") is True
            or provider_name == "mock"
        ):
            args.append("--use-sample-tasks")
        hf_repo = extra.get("hf_repo")
        if isinstance(hf_repo, str) and hf_repo.strip():
            args.extend(["--hf-repo", hf_repo.strip()])
        split = extra.get("split")
        if isinstance(split, str) and split.strip():
            args.extend(["--split", split.strip()])
        if extra.get("no_traces") is True:
            args.append("--no-traces")
        _append_scenario_control_flags(args, extra)
        return args

    def _visualwebbench_result(output_dir: Path) -> Path:
        return output_dir / "visualwebbench-results.json"

    def _vision_language_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        benchmark = str(extra.get("vision_benchmark") or extra.get("sub_benchmark") or "textvqa").strip().lower()
        if benchmark not in {"textvqa", "docvqa", "chartqa", "screenspot", "osworld"}:
            benchmark = "textvqa"
        tier = str(extra.get("tier") or "eliza-1-9b").strip() or "eliza-1-9b"
        samples = extra.get("samples")
        sample_count = str(samples if isinstance(samples, int) and samples > 0 else 5)
        args = [
            "bun",
            "run",
            "src/runner.ts",
            "--tier",
            tier,
            "--benchmark",
            benchmark,
            "--samples",
            sample_count,
            "--output",
            str(output_dir / "vision-language-results.json"),
        ]
        agent = str(extra.get("agent") or extra.get("harness") or "eliza").strip().lower()
        if agent in {"eliza", "hermes", "openclaw"}:
            args.extend(["--harness", agent])
        provider_name = str(extra.get("model_provider") or model.provider or "").strip().lower()
        if provider_name:
            args.extend(["--model-provider", provider_name])
        model_name = str(extra.get("model") or model.model or "").strip()
        if model_name:
            args.extend(["--model", model_name])
        if extra.get("smoke") is True:
            args.append("--smoke")
        if extra.get("stub") is True or (model.provider or "").strip().lower() == "mock":
            args.append("--stub")
        _append_scenario_control_flags(args, extra)
        return args

    def _vision_language_result(output_dir: Path) -> Path:
        return output_dir / "vision-language-results.json"

    def _rlm_bench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for RLM benchmark.

        Supports S-NIAH (Streaming Needle-in-a-Haystack) and OOLONG benchmarks
        from the RLM paper (arXiv:2512.24601).
        """
        args = [
            python,
            repo("benchmarks/rlm-bench/run_benchmark.py"),
            "--output-dir",
            str(output_dir),
        ]
        mode = extra.get("mode")
        if isinstance(mode, str) and mode in ("stub", "rlm", "eliza", "custom"):
            args.extend(["--mode", mode])
        else:
            # Default to a REAL mode (#9475): "rlm" runs real RLM-plugin
            # inference (no Eliza server needed). The orchestrator's adapter
            # default_extra_config overrides this to the fuller "eliza" agent
            # loop; only the bare/no-extra path falls back here. "stub" is the
            # heuristic mock and must be requested explicitly via extra.
            args.extend(["--mode", "rlm"])
        backend = extra.get("backend")
        if isinstance(backend, str):
            args.extend(["--backend", backend])
        context_lengths = extra.get("context_lengths")
        if isinstance(context_lengths, str):
            args.extend(["--context-lengths", context_lengths])
        elif isinstance(context_lengths, list) and all(isinstance(x, int) for x in context_lengths):
            args.extend(["--context-lengths", ",".join(str(x) for x in cast(list[int], context_lengths))])
        tasks_per_config = extra.get("tasks_per_config")
        if isinstance(tasks_per_config, int) and tasks_per_config > 0:
            args.extend(["--tasks-per-config", str(tasks_per_config)])
        max_iterations = extra.get("max_iterations")
        if isinstance(max_iterations, int) and max_iterations > 0:
            args.extend(["--max-iterations", str(max_iterations)])
        max_depth = extra.get("max_depth")
        if isinstance(max_depth, int) and max_depth > 0:
            args.extend(["--max-depth", str(max_depth)])
        dual_model = extra.get("dual_model")
        if dual_model is True:
            args.append("--dual-model")
        no_s_niah = extra.get("no_s_niah")
        if no_s_niah is True:
            args.append("--no-s-niah")
        no_oolong = extra.get("no_oolong")
        if no_oolong is True:
            args.append("--no-oolong")
        if model.model:
            args.extend(["--root-model", model.model, "--subcall-model", model.model])
        _append_scenario_control_flags(args, extra)
        return args

    def _rlm_bench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="rlm_bench_results_*.json")

    def _solana_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for Solana gym benchmark.

        The explorer is env-driven (no CLI flags). Caller propagates settings
        via environment variables: ``MODEL_NAME``, ``MAX_MESSAGES``,
        ``ENVIRONMENT_CONFIG``, ``USE_EXTERNAL_SURFPOOL``, and ``OUTPUT_DIR``.
        """
        args = [
            python, "-m", "benchmarks.solana.eliza_explorer", "--output-dir", str(output_dir),
        ]
        harness = extra.get("agent") or extra.get("harness")
        if isinstance(harness, str) and harness.strip():
            args.extend(["--harness", harness.strip().lower()])
        _append_scenario_control_flags(args, extra)
        # All knobs flow through env vars read by ``eliza_explorer.main``.
        _ = model
        return args

    def _solana_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="eliza_*_metrics.json")

    def _osworld_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for OSWorld benchmark."""
        args = [
            python,
            repo("benchmarks/OSWorld/scripts/python/run_multienv_eliza.py"),
            "--result_dir",
            str(output_dir),
        ]
        if model.model:
            args.extend(["--model", model.model])
        provider = extra.get("provider_name")
        if isinstance(provider, str):
            args.extend(["--provider_name", provider])
        else:
            args.extend(["--provider_name", "docker"])
        path_to_vm = extra.get("path_to_vm")
        if isinstance(path_to_vm, str):
            args.extend(["--path_to_vm", path_to_vm])
        observation = extra.get("observation_type")
        if isinstance(observation, str):
            args.extend(["--observation_type", observation])
        else:
            args.extend(["--observation_type", "screenshot_a11y_tree"])
        action_space = extra.get("action_space")
        if isinstance(action_space, str):
            args.extend(["--action_space", action_space])
        max_steps = extra.get("max_steps")
        if isinstance(max_steps, int) and max_steps > 0:
            args.extend(["--max_steps", str(max_steps)])
        else:
            args.extend(["--max_steps", "3"])
        a11y_tree_max_tokens = extra.get("a11y_tree_max_tokens")
        if isinstance(a11y_tree_max_tokens, int) and a11y_tree_max_tokens > 0:
            args.extend(["--a11y_tree_max_tokens", str(a11y_tree_max_tokens)])
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max_tasks", str(max_tasks)])
        task_id = extra.get("task_id")
        if isinstance(task_id, str):
            args.extend(["--task_id", task_id])
        domain = extra.get("domain")
        if isinstance(domain, str):
            args.extend(["--domain", domain])
        headless = extra.get("headless")
        if headless is True:
            args.append("--headless")
        dry_run = extra.get("dry_run")
        if dry_run is True:
            _validate_osworld_dry_run_label(extra)
            args.append("--dry_run")
        _append_scenario_control_flags(args, extra)
        _ = model
        return args

    def _osworld_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="osworld-eliza-results-*.json")

    # HyperliquidBench - perp-trading plan generation + Rust execution
    def _hyperliquid_bench_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        """Build command for HyperliquidBench.

        Defaults to the eliza TypeScript bridge. Set ``extra.agent`` to
        ``deterministic`` or ``python`` for the local deterministic smoke path.
        Always runs in ``--demo`` mode unless the caller
        explicitly opts in to ``--no-demo`` (which requires ``HL_PRIVATE_KEY``
        and a non-mainnet network).
        """
        args = [
            python,
            "-m",
            "benchmarks.HyperliquidBench",
            "--output",
            str(output_dir),
        ]
        agent = extra.get("agent")
        provider_name = (model.provider or "").strip().lower()
        if agent in {"deterministic", "python"} or extra.get("mock") is True or provider_name == "mock":
            args.extend(["--mode", "deterministic"])
        else:
            args.extend(["--mode", "eliza"])

        if model.model:
            args.extend(["--model", model.model])
        if model.temperature is not None:
            args.extend(["--temperature", str(model.temperature)])

        coins = extra.get("coins")
        if isinstance(coins, list):
            coin_values = [str(c) for c in coins if str(c).strip()]
            if coin_values:
                args.extend(["--coins", ",".join(coin_values)])
        elif isinstance(coins, str) and coins.strip():
            args.extend(["--coins", coins.strip()])

        max_steps = extra.get("max_steps")
        if isinstance(max_steps, int) and max_steps > 0:
            args.extend(["--max-steps", str(max_steps)])
        max_iterations = extra.get("max_iterations")
        if isinstance(max_iterations, int) and max_iterations > 0:
            args.extend(["--max-iterations", str(max_iterations)])
        _append_scenario_control_flags(args, extra)

        builder_code = extra.get("builder_code")
        if isinstance(builder_code, str) and builder_code.strip():
            args.extend(["--builder-code", builder_code.strip()])

        tasks = extra.get("tasks")
        if isinstance(tasks, list) and tasks:
            args.append("--tasks")
            args.extend(str(t) for t in tasks)
        elif extra.get("coverage") is True:
            args.append("--coverage")

        # Network + demo handling. Default behavior is demo=true with testnet.
        network_raw = extra.get("network")
        network = network_raw.strip().lower() if isinstance(network_raw, str) else "testnet"
        if network in {"testnet", "mainnet", "local"}:
            args.extend(["--network", network])

        if extra.get("no_demo") is True or extra.get("demo") is False:
            # Live trading on the chosen network — caller must have HL_PRIVATE_KEY.
            args.append("--no-demo")
        # else: --demo is the default in __main__.py, no flag needed

        return args

    def _hyperliquid_bench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="hyperliquid_bench-*.json")

    def _gauntlet_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for Solana Gauntlet benchmark with harness bridge agents."""
        agent = str(extra.get("agent") or extra.get("harness") or "eliza").strip().lower()
        if agent == "python":
            agent_path = repo("benchmarks/gauntlet/agents/eliza_agent.py")
        elif agent == "hermes":
            agent_path = repo("benchmarks/gauntlet/agents/hermes_bridge_agent.py")
        elif agent == "openclaw":
            agent_path = repo("benchmarks/gauntlet/agents/openclaw_bridge_agent.py")
        else:
            agent_path = repo("benchmarks/gauntlet/agents/eliza_bridge_agent.py")

        args = [
            python,
            "-m",
            "gauntlet.cli",
            "run",
            "--agent",
            agent_path,
            "--scenarios",
            repo("benchmarks/gauntlet/scenarios"),
            "--programs",
            repo("benchmarks/gauntlet/programs"),
            "--output",
            str(output_dir),
        ]
        clone_mainnet = extra.get("clone_mainnet")
        if clone_mainnet is True:
            args.append("--clone-mainnet")
        elif extra.get("mock") is True or (model.provider or "").strip().lower() == "mock":
            args.append("--mock")
        seed = extra.get("seed")
        if isinstance(seed, int) and seed > 0:
            args.extend(["--seed", str(seed)])
        max_scenarios = extra.get("max_scenarios")
        if not isinstance(max_scenarios, int):
            max_scenarios = extra.get("max_tasks")
        if isinstance(max_scenarios, int) and max_scenarios > 0:
            args.extend(["--max-scenarios", str(max_scenarios)])
        _append_scenario_control_flags(args, extra)
        return args

    def _gauntlet_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="*.json")

    # ClawBench - OpenClaw agent evaluation via the eliza benchmark bridge
    def _clawbench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for ClawBench scenario evaluation through the eliza bridge.

        Routes through ``clawbench/eliza_adapter.py`` which honors the shared
        ``ELIZA_BENCH_URL`` / ``ELIZA_BENCH_TOKEN`` env vars so all eliza-bridge
        benchmarks reuse the same server. Output filename matches
        ``_clawbench_result``'s ``trajectory_*.json`` glob.
        """
        agent = str(extra.get("agent") or extra.get("harness") or "eliza").strip().lower()
        scenario = extra.get("scenario")
        scenario_name = scenario.strip() if isinstance(scenario, str) and scenario.strip() else "inbox_triage"
        if agent in {"eliza", "hermes", "openclaw", "smithers"}:
            output_path = output_dir / f"trajectory_{scenario_name}.json"
            args = [
                python,
                "-m",
                "clawbench.multi_harness_runner",
                "--harness",
                agent,
                "--scenario",
                scenario_name,
                "--model",
                model.model or "gemma-4-31b",
                "--output",
                str(output_path),
                "--json",
            ]
            return args
        args = [
            python,
            repo("benchmarks/clawbench/eliza_adapter.py"),
            "--output-dir",
            str(output_dir),
        ]
        if isinstance(scenario, str) and scenario.strip():
            args.extend(["--scenario", scenario.strip()])
        else:
            args.extend(["--scenario", "inbox_triage"])
        variant = extra.get("variant")
        if isinstance(variant, str) and variant.strip():
            args.extend(["--variant", variant.strip()])
        if extra.get("start_server") is True:
            args.append("--start-server")
        _ = model
        return args

    def _clawbench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="trajectory_*.json")

    # OpenClaw Benchmark - AI assistant coding tasks
    def _openclaw_bench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for OpenClaw benchmark tasks."""
        args = [
            python,
            repo("benchmarks/openclaw-benchmark/eliza_adapter.py"),
            "--json",
            "--output-dir",
            str(output_dir),
        ]
        mode = extra.get("mode")
        provider_name = (model.provider or "").strip().lower()
        if extra.get("mock") is True or provider_name == "mock":
            # Conceptual standalone mode is the only no-key path in this
            # adapter. It is useful for smoke/readiness, not scored parity.
            args.extend(["--mode", "conceptual"])
        elif isinstance(mode, str) and mode.strip() in {"execution", "conceptual"}:
            args.extend(["--mode", mode.strip()])
        else:
            # Default to execution: real file/exec validation, not keyword matching.
            args.extend(["--mode", "execution"])
        task = extra.get("task")
        if isinstance(task, str) and task.strip():
            args.extend(["--task", task.strip()])
        elif extra.get("all") is True:
            args.append("--all")
        else:
            args.extend(["--task", "setup"])
        if model.model:
            args.extend(["--model", model.model])
        if extra.get("docker") is True:
            args.append("--docker")
        if extra.get("start_server") is True:
            args.append("--start-server")
        return args

    def _openclaw_bench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="openclaw_*_exec_*.json")

    # ConfigBench - secrets + plugin-manager security benchmark (Bun runtime)
    def _configbench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for ConfigBench (Bun TS).

        ConfigBench instantiates ``@elizaos/core`` in-process when ``--eliza``
        is set, so it does not route through the TS bridge. The ``eliza`` agent
        path requires a provider key (GROQ/OPENAI); other paths are oracle/random.
        """
        args = [
            "bun",
            "run",
            "src/index.ts",
            "--output",
            str(output_dir),
        ]
        agent = extra.get("agent")
        if isinstance(agent, str) and agent.strip().lower() in {"hermes", "openclaw"}:
            raise ValueError("ConfigBench only supports the Eliza handler today")
        if agent == "eliza" or (model.provider or "").strip().lower() == "eliza":
            args.append("--eliza")
        elif extra.get("eliza") is True:
            args.append("--eliza")
        limit = extra.get("limit")
        if isinstance(limit, int) and limit > 0:
            args.extend(["--limit", str(limit)])
        if extra.get("verbose") is True:
            args.append("--verbose")
        return args

    def _configbench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="configbench-results-*.json")

    # VoiceBench - end-to-end voice latency benchmark (Bun TS via run.sh)
    def _voicebench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for VoiceBench.

        VoiceBench instantiates the elizaOS TS runtime in-process and is wrapped
        by ``run.sh`` which manages provider env defaults, audio fixture
        resolution, and dataset manifest selection. Run from the voicebench
        directory (``cwd_rel`` resolves there).
        """
        args = ["bash", "./run.sh", f"--output-dir={output_dir}"]
        profile_raw = extra.get("profile")
        provider_name = (model.provider or "").strip().lower()
        if isinstance(profile_raw, str) and profile_raw.strip():
            profile = profile_raw.strip().lower()
        elif os.environ.get("VOICEBENCH_PROFILE", "").strip().lower():
            profile = os.environ["VOICEBENCH_PROFILE"].strip().lower()
        elif extra.get("mock") is True or provider_name == "mock":
            profile = "mock"
        elif provider_name == "cerebras" and os.environ.get("CEREBRAS_API_KEY"):
            profile = "local-cerebras"
        elif provider_name == "elevenlabs":
            profile = "elevenlabs"
        else:
            profile = "groq"
        if profile not in {"groq", "elevenlabs", "mock", "local-cerebras", "local-eliza1"}:
            raise ValueError(
                f"voicebench: unsupported profile '{profile}' "
                "(expected groq, elevenlabs, local-cerebras, local-eliza1, or mock)"
            )
        args.append(f"--profile={profile}")
        iterations = extra.get("iterations")
        if isinstance(iterations, int) and iterations > 0:
            args.append(f"--iterations={iterations}")
        dataset = extra.get("dataset")
        if isinstance(dataset, str) and dataset.strip():
            args.append(f"--dataset={dataset.strip()}")
        else:
            env_dataset = (
                os.environ.get("VOICEBENCH_DATASET")
                or os.environ.get("VOICEBENCH_DATASET_PATH")
                or ""
            ).strip()
            if env_dataset:
                args.append(f"--dataset={env_dataset}")
        _ = model
        return args

    def _voicebench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="voicebench-typescript-*.json")

    # Audio MMAU - audio MCQ benchmark (Sakshi et al., ICLR 2025).
    # Not Salesforce's agent MMAU (arXiv:2407.18961).
    def _mmau_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build the elizaos-mmau-audio CLI invocation.

        Routes through the Python-native Audio MMAU package. Pure MCQ -- no
        LLM-judge dispatch. Real harness rows stream upstream HF audio by
        default; mock/fixture mode remains explicit for smoke runs.
        """
        args = [
            python,
            "-m",
            "elizaos_mmau_audio",
            "--output",
            str(output_dir),
            "--no-traces",
            "--json",
        ]
        provider_name = (model.provider or "").strip().lower()
        agent_raw = extra.get("agent")
        if isinstance(agent_raw, str) and agent_raw.strip():
            agent = agent_raw.strip().lower()
        elif provider_name in {"eliza", "hermes", "openclaw", "mock"}:
            agent = provider_name
        else:
            agent = "mock"
        if agent == "mock":
            args.append("--mock")
        else:
            args.extend(["--agent", agent])
        split_raw = extra.get("split")
        if isinstance(split_raw, str) and split_raw.strip():
            args.extend(["--split", split_raw.strip()])
        category_raw = extra.get("category")
        if isinstance(category_raw, str) and category_raw.strip():
            args.extend(["--category", category_raw.strip()])
        limit_raw = extra.get("limit")
        if isinstance(limit_raw, int) and limit_raw > 0:
            args.extend(["--limit", str(limit_raw)])
        if extra.get("hf") is True or (agent != "mock" and extra.get("fixture") is not True):
            args.append("--hf")
        if model.model:
            args.extend(["--model", model.model])
        if model.provider:
            args.extend(["--provider", model.provider])
        _append_scenario_control_flags(args, extra)
        return args

    def _mmau_result(output_dir: Path) -> Path:
        return output_dir / "mmau-results.json"

    # VoiceBench-quality - vendored upstream VoiceBench (Chen et al. 2024)
    def _voicebench_quality_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        """Build the elizaos-voicebench CLI invocation.

        ``model.model`` selects the agent backend (``eliza``/``hermes``/
        ``openclaw``/``echo``). Defaults to ``eliza`` for real harness
        runs. Provider ``mock``, explicit ``mock``, or the ``echo`` agent
        flips on ``--mock`` so smoke runs remain opt-in.
        """
        agent_raw = extra.get("agent")
        if isinstance(agent_raw, str) and agent_raw.strip():
            agent = agent_raw.strip()
        elif model.model in {"eliza", "hermes", "openclaw", "echo"}:
            agent = str(model.model)
        else:
            agent = "eliza"
        provider_name = (model.provider or "").strip().lower()
        mock_flag = bool(extra.get("mock")) or provider_name == "mock" or agent == "echo"
        args = [
            python,
            "-m",
            "elizaos_voicebench",
            "--agent",
            agent,
            "--output",
            str(output_dir),
        ]
        suite_raw = extra.get("suite")
        if isinstance(suite_raw, str) and suite_raw.strip():
            args.extend(["--suite", suite_raw.strip()])
        limit = extra.get("limit")
        if isinstance(limit, int) and limit > 0:
            args.extend(["--limit", str(limit)])
        judge_model = extra.get("judge_model")
        if isinstance(judge_model, str) and judge_model.strip():
            args.extend(["--judge-model", judge_model.strip()])
        stt_provider = extra.get("stt_provider")
        if isinstance(stt_provider, str) and stt_provider.strip():
            args.extend(["--stt-provider", stt_provider.strip()])
        else:
            env_stt_provider = (
                os.environ.get("VOICEBENCH_QUALITY_STT_PROVIDER")
                or os.environ.get("VOICEBENCH_STT_PROVIDER")
                or ""
            ).strip()
            if env_stt_provider:
                args.extend(["--stt-provider", env_stt_provider])
        if extra.get("fixtures") is True:
            args.append("--fixtures")
        if mock_flag:
            args.append("--mock")
        _append_scenario_control_flags(args, extra)
        return args

    def _voicebench_quality_result(output_dir: Path) -> Path:
        return output_dir / "voicebench-quality-results.json"

    # Social-Alpha - trust-marketplace benchmark on real Discord crypto chat data
    def _social_alpha_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for Social-Alpha.

        Routes through the click-based ``benchmark.harness`` CLI installed by
        the ``trust-marketplace-benchmark`` pyproject. Defaults to the bundled
        smoke fixture when the full ``trenches-chat-dataset`` checkout is not
        present; callers can override via ``data_dir``. Defaults to the
        ``baseline`` system unless another system is requested via
        ``extra.system`` or ``model.provider``.
        """
        data_dir_raw = extra.get("data_dir")
        if isinstance(data_dir_raw, str) and data_dir_raw.strip():
            data_dir = data_dir_raw.strip()
        else:
            full_data_dir = repo_root / "benchmarks/social-alpha/trenches-chat-dataset/data"
            data_dir = "trenches-chat-dataset/data" if full_data_dir.exists() else "fixtures/smoke-data"
        args = [
            python,
            "-m",
            "benchmark.harness",
            "--data-dir",
            data_dir,
            "--output",
            str(output_dir),
        ]
        system_raw = extra.get("system")
        if isinstance(system_raw, str) and system_raw.strip():
            system = system_raw.strip()
        else:
            provider_lower = (model.provider or "").strip().lower()
            if provider_lower in {"eliza", "eliza-bridge", "eliza-ts"}:
                system = "eliza-bridge"
            elif provider_lower in {"cerebras", "openai", "groq", "openrouter", "vllm"}:
                system = "full"
            else:
                system = "baseline"
        args.extend(["--system", system])
        if model.model:
            args.extend(["--model", model.model])
        api_base = extra.get("api_base")
        if isinstance(api_base, str) and api_base.strip():
            args.extend(["--api-base", api_base.strip()])
        suites = extra.get("suites")
        if isinstance(suites, list):
            for suite in suites:
                if isinstance(suite, str) and suite.strip():
                    args.extend(["--suite", suite.strip()])
        elif isinstance(suites, str) and suites.strip():
            args.extend(["--suite", suites.strip()])
        if extra.get("generate_gt") is True:
            args.append("--generate-gt")
        _append_scenario_control_flags(args, extra)
        return args

    def _social_alpha_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="benchmark_results_*.json")

    def _trust_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        handler_raw = extra.get("handler")
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        provider_name = (model.provider or "").strip().lower()
        if extra.get("mock") is True or provider_name == "mock":
            handler = "oracle"
        elif isinstance(handler_raw, str) and handler_raw.strip():
            handler = handler_raw.strip()
        elif agent in {"eliza", "hermes", "openclaw"} or provider_name in {"cerebras", "openai", "groq", "openrouter", "vllm"}:
            handler = "eliza"
        else:
            handler = "oracle"
        args = [
            python,
            repo("benchmarks/trust/run_benchmark.py"),
            "--handler",
            handler,
            "--output",
            str(output_dir / "trust-results.json"),
        ]
        if handler in {"eliza", "llm"}:
            if model.provider:
                args.extend(["--model-provider", model.provider])
            if model.model:
                args.extend(["--model", model.model])
        categories = extra.get("categories")
        if isinstance(categories, list) and all(isinstance(x, str) for x in categories):
            args.extend(["--categories", *cast(list[str], categories)])
        difficulty = extra.get("difficulty")
        if isinstance(difficulty, list) and all(isinstance(x, str) for x in difficulty):
            args.extend(["--difficulty", *cast(list[str], difficulty)])
        tags = extra.get("tags")
        if isinstance(tags, list) and all(isinstance(x, str) for x in tags):
            args.extend(["--tags", *cast(list[str], tags)])
        threshold = extra.get("threshold")
        if isinstance(threshold, (int, float)):
            args.extend(["--threshold", str(float(threshold))])
        return args

    def _trust_result(output_dir: Path) -> Path:
        return output_dir / "trust-results.json"

    # WebShop - product-search/purchase benchmark with Eliza agent
    def _webshop_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        """Build command for WebShop benchmark.

        Real harness rows use the upstream WebShop data loader (small profile
        by default, auto-fetched when absent). The bundled sample catalog is
        smoke-only and must be requested explicitly or via mock mode.
        """
        args = [
            python,
            "-m",
            "elizaos_webshop",
            "--output",
            str(output_dir),
        ]
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        provider_lower = (model.provider or "").strip().lower()
        if extra.get("mock") is True or provider_lower == "mock":
            args.append("--mock")
        else:
            args.append("--bridge")
            if model.provider and provider_lower not in {"eliza", "eliza-bridge", "eliza-ts"}:
                args.extend(["--model-provider", model.provider])
        if model.model:
            args.extend(["--model", model.model])
        provider_name = (model.provider or "").strip().lower()
        if extra.get("mock") is True or provider_name == "mock":
            args.append("--mock")
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        max_turns = extra.get("max_turns")
        if isinstance(max_turns, int) and max_turns > 0:
            args.extend(["--max-turns", str(max_turns)])
        trials = extra.get("trials")
        if isinstance(trials, int) and trials > 0:
            args.extend(["--trials", str(trials)])
        profile = extra.get("profile")
        if isinstance(profile, str) and profile.strip() in {"small", "full"}:
            args.extend(["--profile", profile.strip()])
        if extra.get("hf") is True:
            args.append("--hf")
            split = extra.get("split")
            if isinstance(split, str) and split.strip():
                args.extend(["--split", split.strip()])
        elif (
            extra.get("sample") is True
            or extra.get("use_sample_tasks") is True
            or extra.get("mock") is True
            or provider_name == "mock"
        ):
            args.append("--sample")
        if extra.get("trajectories") is True:
            args.append("--trajectories")
        else:
            args.append("--no-trajectories")
        if model.temperature is not None:
            args.extend(["--temperature", str(model.temperature)])
        _append_scenario_control_flags(args, extra)
        return args

    def _webshop_result(output_dir: Path) -> Path:
        return output_dir / "webshop-results.json"

    # WooBench - mystical-reading conversation benchmark.
    def _woobench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = [
            python,
            "-m",
            "benchmarks.woobench",
            "--output",
            str(output_dir),
        ]
        if model.model:
            args.extend(["--model", model.model])

        agent_raw = extra.get("agent")
        agent = str(agent_raw or extra.get("harness") or "").strip().lower()
        provider_lower = (model.provider or "").strip().lower()
        payment_mode = extra.get("payment") is True or extra.get("payments") is True
        if agent_raw == "dummy" or extra.get("mock") is True or provider_lower == "mock":
            args.extend(["--agent", "dummy-charge" if payment_mode else "dummy"])
        elif agent in {"eliza", "hermes", "openclaw", "smithers"}:
            args.extend(["--agent", agent])
        else:
            args.extend(["--agent", "eliza"])

        evaluator = extra.get("evaluator")
        if isinstance(evaluator, str) and evaluator in {"llm", "heuristic"}:
            args.extend(["--evaluator", evaluator])
        elif agent_raw == "dummy" or extra.get("mock") is True or provider_lower == "mock":
            args.extend(["--evaluator", "heuristic"])

        scenario = extra.get("scenario")
        if isinstance(scenario, str) and scenario.strip():
            args.extend(["--scenario", scenario.strip()])
        scenarios = extra.get("scenarios")
        if isinstance(scenarios, list) and all(isinstance(x, str) for x in scenarios):
            args.extend(["--scenarios", ",".join(cast(list[str], scenarios))])
        system = extra.get("system")
        if isinstance(system, str) and system.strip():
            args.extend(["--system", system.strip()])
        persona = extra.get("persona")
        if isinstance(persona, str) and persona.strip():
            args.extend(["--persona", persona.strip()])
        concurrency = extra.get("concurrency")
        if isinstance(concurrency, int) and concurrency > 0:
            args.extend(["--concurrency", str(concurrency)])
        payment_mock_url = extra.get("payment_mock_url")
        if isinstance(payment_mock_url, str) and payment_mock_url.strip():
            args.extend(["--payment-mock-url", payment_mock_url.strip()])
        _append_scenario_control_flags(args, extra)
        return args

    def _woobench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="woobench_*.json")

    # scambench
    def _scambench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        provider = (model.provider or "").strip().lower() or "vllm"
        args = [
            python,
            "-m",
            "benchmarks.scambench.cli",
            "--provider",
            provider,
            "--out",
            str(output_dir),
        ]
        if model.model:
            args.extend(["--model", model.model])
        base_url = extra.get("base_url") or extra.get("vllm_base_url")
        if isinstance(base_url, str) and base_url.strip():
            args.extend(["--base-url", base_url.strip()])
        api_key_env = extra.get("api_key_env")
        if isinstance(api_key_env, str) and api_key_env.strip():
            args.extend(["--api-key-env", api_key_env.strip()])
        datasets = extra.get("dataset")
        if isinstance(datasets, list):
            for d in datasets:
                if isinstance(d, str) and d.strip():
                    args.extend(["--dataset", d.strip()])
        elif isinstance(datasets, str) and datasets.strip():
            args.extend(["--dataset", datasets.strip()])
        max_examples = extra.get("max_examples")
        if isinstance(max_examples, int) and max_examples > 0:
            args.extend(["--max-examples", str(max_examples)])
        max_new_tokens = extra.get("max_new_tokens")
        if isinstance(max_new_tokens, int) and max_new_tokens > 0:
            args.extend(["--max-new-tokens", str(max_new_tokens)])
        temperature = extra.get("temperature")
        if isinstance(temperature, (int, float)) and not isinstance(temperature, bool):
            args.extend(["--temperature", str(float(temperature))])
        _append_scenario_control_flags(args, extra)
        return args

    def _scambench_result(output_dir: Path) -> Path:
        return output_dir / "scambench-results.json"

    # abliteration-robustness
    def _abliteration_robustness_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        provider = (model.provider or "").strip().lower() or "vllm"
        args = [
            python,
            "-m",
            "benchmarks.abliteration-robustness.cli",
            "--provider",
            provider,
            "--out",
            str(output_dir),
        ]
        if model.model:
            args.extend(["--model", model.model])
        base_url = extra.get("base_url")
        if not base_url and provider == "vllm":
            base_url = extra.get("vllm_base_url") or "http://127.0.0.1:8001/v1"
        if isinstance(base_url, str) and base_url.strip():
            args.extend(["--base-url", base_url.strip()])
        api_key_env = extra.get("api_key_env")
        if isinstance(api_key_env, str) and api_key_env.strip():
            args.extend(["--api-key-env", api_key_env.strip()])
        dataset = extra.get("dataset")
        if isinstance(dataset, str) and dataset.strip():
            args.extend(["--dataset", dataset.strip()])
        dataset_path = extra.get("dataset_path")
        if isinstance(dataset_path, str) and dataset_path.strip():
            args.extend(["--dataset-path", dataset_path.strip()])
        max_examples = extra.get("max_examples")
        if isinstance(max_examples, int) and max_examples > 0:
            args.extend(["--max-examples", str(max_examples)])
        max_new_tokens = extra.get("max_new_tokens")
        if isinstance(max_new_tokens, int) and max_new_tokens > 0:
            args.extend(["--max-new-tokens", str(max_new_tokens)])
        temperature = extra.get("temperature")
        if isinstance(temperature, (int, float)) and not isinstance(temperature, bool) and temperature >= 0:
            args.extend(["--temperature", str(float(temperature))])
        tool_choice = extra.get("tool_choice")
        if isinstance(tool_choice, str) and tool_choice in {"auto", "required", "none"}:
            args.extend(["--tool-choice", tool_choice])
        else:
            args.extend(["--tool-choice", "none"])
        return args

    def _abliteration_robustness_result(output_dir: Path) -> Path:
        return output_dir / "abliteration-robustness-results.json"

    # action-calling
    def _action_calling_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        provider = (model.provider or "").strip().lower() or "vllm"
        agent = str(extra.get("agent") or extra.get("harness") or "").strip().lower()
        if extra.get("mock") is True or provider == "mock":
            provider = "mock"
        elif agent in {"eliza", "hermes", "openclaw"}:
            provider = agent
        args = [
            python,
            "-m",
            "benchmarks.action-calling.cli",
            "--provider",
            provider,
            "--out",
            str(output_dir),
        ]
        if model.model:
            args.extend(["--model", model.model])
        base_url = extra.get("base_url")
        if isinstance(base_url, str) and base_url.strip():
            args.extend(["--base-url", base_url.strip()])
        api_key_env = extra.get("api_key_env")
        if isinstance(api_key_env, str) and api_key_env.strip():
            args.extend(["--api-key-env", api_key_env.strip()])
        test_file = extra.get("test_file")
        if isinstance(test_file, str) and test_file.strip():
            args.extend(["--test-file", test_file.strip()])
        max_examples = extra.get("max_examples")
        if isinstance(max_examples, int) and max_examples > 0:
            args.extend(["--max-examples", str(max_examples)])
        else:
            args.extend(["--max-examples", "100"])
        max_new_tokens = extra.get("max_new_tokens")
        if isinstance(max_new_tokens, int) and max_new_tokens > 0:
            args.extend(["--max-new-tokens", str(max_new_tokens)])
        temperature = extra.get("temperature")
        if isinstance(temperature, (int, float)) and not isinstance(temperature, bool) and temperature >= 0:
            args.extend(["--temperature", str(float(temperature))])
        tool_choice = extra.get("tool_choice")
        if isinstance(tool_choice, str) and tool_choice in {"auto", "required", "none"}:
            args.extend(["--tool-choice", tool_choice])
        _append_scenario_control_flags(args, extra)
        return args

    def _action_calling_result(output_dir: Path) -> Path:
        return output_dir / "action-calling-results.json"

    # ----- standard public benchmarks (MMLU / HumanEval / GSM8K / MT-Bench) -----

    def _standard_bench_base_args(
        module: str,
        output_dir: Path,
        model: ModelSpec,
        extra: Mapping[str, JSONValue],
    ) -> list[str]:
        """Shared CLI arg builder for ``benchmarks.standard.<name>`` runners.

        Forwards ``--model-endpoint`` / ``--provider`` / ``--model`` /
        ``--api-key-env`` / ``--mock`` / ``--limit`` from ``ModelSpec`` +
        ``extra`` to the standardized adapter CLI.
        """

        args: list[str] = [
            python,
            "-m",
            module,
            "--output",
            str(output_dir),
        ]
        if model.model:
            args.extend(["--model", model.model])
        endpoint = extra.get("model_endpoint") or extra.get("base_url")
        provider_name = (model.provider or "").strip().lower()
        if isinstance(endpoint, str) and endpoint.strip():
            args.extend(["--model-endpoint", endpoint.strip()])
        elif provider_name and provider_name != "mock":
            args.extend(["--provider", provider_name])
        api_key_env = extra.get("api_key_env")
        if isinstance(api_key_env, str) and api_key_env.strip():
            args.extend(["--api-key-env", api_key_env.strip()])
        limit = extra.get("limit")
        if isinstance(limit, int) and limit > 0:
            args.extend(["--limit", str(limit)])
        if extra.get("mock") is True or provider_name == "mock":
            args.append("--mock")
        return args

    def _mmlu_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = _standard_bench_base_args("benchmarks.standard.mmlu", output_dir, model, extra)
        max_tokens = extra.get("max_tokens")
        if isinstance(max_tokens, int) and max_tokens > 0:
            args.extend(["--max-tokens", str(max_tokens)])
        return args

    def _mmlu_result(output_dir: Path) -> Path:
        return output_dir / "mmlu-results.json"

    def _humaneval_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = _standard_bench_base_args("benchmarks.standard.humaneval", output_dir, model, extra)
        max_tokens = extra.get("max_tokens")
        if isinstance(max_tokens, int) and max_tokens > 0:
            args.extend(["--max-tokens", str(max_tokens)])
        timeout_s = extra.get("timeout_s")
        if isinstance(timeout_s, (int, float)) and not isinstance(timeout_s, bool) and timeout_s > 0:
            args.extend(["--timeout-s", str(float(timeout_s))])
        return args

    def _humaneval_result(output_dir: Path) -> Path:
        return output_dir / "humaneval-results.json"

    def _gsm8k_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = _standard_bench_base_args("benchmarks.standard.gsm8k", output_dir, model, extra)
        max_tokens = extra.get("max_tokens")
        if isinstance(max_tokens, int) and max_tokens > 0:
            args.extend(["--max-tokens", str(max_tokens)])
        return args

    def _gsm8k_result(output_dir: Path) -> Path:
        return output_dir / "gsm8k-results.json"

    def _mt_bench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        args = _standard_bench_base_args("benchmarks.standard.mt_bench", output_dir, model, extra)
        judge_endpoint = extra.get("judge_endpoint")
        if isinstance(judge_endpoint, str) and judge_endpoint.strip():
            args.extend(["--judge-endpoint", judge_endpoint.strip()])
        judge_provider = extra.get("judge_provider")
        if isinstance(judge_provider, str) and judge_provider.strip():
            args.extend(["--judge-provider", judge_provider.strip()])
        judge_model = extra.get("judge_model")
        if isinstance(judge_model, str) and judge_model.strip():
            args.extend(["--judge-model", judge_model.strip()])
        judge_api_key_env = extra.get("judge_api_key_env")
        if isinstance(judge_api_key_env, str) and judge_api_key_env.strip():
            args.extend(["--judge-api-key-env", judge_api_key_env.strip()])
        max_tokens = extra.get("max_tokens")
        if isinstance(max_tokens, int) and max_tokens > 0:
            args.extend(["--max-tokens", str(max_tokens)])
        temperature = extra.get("temperature")
        if (
            isinstance(temperature, (int, float))
            and not isinstance(temperature, bool)
            and temperature >= 0
        ):
            args.extend(["--temperature", str(float(temperature))])
        judge_max_tokens = extra.get("judge_max_tokens")
        if isinstance(judge_max_tokens, int) and judge_max_tokens > 0:
            args.extend(["--judge-max-tokens", str(judge_max_tokens)])
        return args

    def _mt_bench_result(output_dir: Path) -> Path:
        return output_dir / "mt-bench-results.json"

    def _trajectory_replay_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        """Build the trajectory-replay CLI invocation.

        Required ``extra`` keys:

        * ``traj_set`` (str): directory of trajectory JSON files.
        * ``baseline`` (str): model id whose recorded outputs are ground truth.

        Optional knobs that map straight onto the adapter flags:
        ``reward_threshold`` (float, default 0.5),
        ``exact_action_sequence`` (bool, default True; set False for set match),
        ``action_weight`` (float, default 0.5),
        ``final_state_weight`` (float, default 0.5),
        ``max_tokens`` (int, default 768).
        """

        args = _standard_bench_base_args(
            "benchmarks.standard.trajectory_replay", output_dir, model, extra
        )
        traj_set = extra.get("traj_set")
        if not isinstance(traj_set, str) or not traj_set.strip():
            raise ValueError(
                "trajectory_replay requires extra.traj_set (directory of trajectory JSON files)"
            )
        args.extend(["--traj-set", traj_set.strip()])
        baseline = extra.get("baseline")
        if not isinstance(baseline, str) or not baseline.strip():
            raise ValueError(
                "trajectory_replay requires extra.baseline (baseline model id)"
            )
        args.extend(["--baseline", baseline.strip()])
        reward_threshold = extra.get("reward_threshold")
        if (
            isinstance(reward_threshold, (int, float))
            and not isinstance(reward_threshold, bool)
        ):
            args.extend(["--reward-threshold", str(float(reward_threshold))])
        exact_action_sequence = extra.get("exact_action_sequence")
        if exact_action_sequence is False:
            args.append("--no-exact-action-sequence")
        elif exact_action_sequence is True:
            args.append("--exact-action-sequence")
        action_weight = extra.get("action_weight")
        if (
            isinstance(action_weight, (int, float))
            and not isinstance(action_weight, bool)
        ):
            args.extend(["--action-weight", str(float(action_weight))])
        final_state_weight = extra.get("final_state_weight")
        if (
            isinstance(final_state_weight, (int, float))
            and not isinstance(final_state_weight, bool)
        ):
            args.extend(["--final-state-weight", str(float(final_state_weight))])
        max_tokens = extra.get("max_tokens")
        if isinstance(max_tokens, int) and max_tokens > 0:
            args.extend(["--max-tokens", str(max_tokens)])
        return args

    def _trajectory_replay_result(output_dir: Path) -> Path:
        return output_dir / "trajectory-replay-results.json"

    # lifeops-bench
    def _lifeops_bench_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        """Build the LifeOpsBench CLI invocation.

        ``model.model`` selects the agent backend: ``perfect`` / ``wrong``
        for hermetic oracle runs; ``hermes`` / ``cerebras-direct`` /
        ``eliza`` for adapter-backed runs that need an API key. Default
        (no model specified) is ``perfect`` for cheap smoke runs.
        """
        agent_raw = extra.get("agent") or extra.get("harness")
        if isinstance(agent_raw, str) and agent_raw.strip():
            agent = agent_raw.strip()
        elif model.model in {"perfect", "wrong", "hermes", "openclaw", "cerebras-direct", "eliza"}:
            agent = str(model.model)
        elif (model.provider or "").strip().lower() in {"cerebras", "openai", "groq", "openrouter", "vllm", "eliza"}:
            agent = "eliza"
        else:
            agent = "perfect"
        args = [
            python,
            "-m",
            "eliza_lifeops_bench",
            "--agent",
            agent,
            "--output-dir",
            str(output_dir),
        ]
        domain = extra.get("domain")
        if isinstance(domain, str) and domain.strip():
            args.extend(["--domain", domain.strip()])
        mode = extra.get("mode")
        if isinstance(mode, str) and mode.strip():
            args.extend(["--mode", mode.strip()])
        scenario = extra.get("scenario")
        if isinstance(scenario, str) and scenario.strip():
            args.extend(["--scenario", scenario.strip()])
        suite = extra.get("suite")
        if isinstance(suite, str) and suite.strip():
            args.extend(["--suite", suite.strip()])
        limit = extra.get("limit")
        if isinstance(limit, int) and limit > 0:
            args.extend(["--limit", str(limit)])
        seeds = extra.get("seeds")
        if isinstance(seeds, int) and seeds > 0:
            args.extend(["--seeds", str(seeds)])
        concurrency = extra.get("concurrency")
        if isinstance(concurrency, int) and concurrency > 0:
            args.extend(["--concurrency", str(concurrency)])
        max_cost_usd = extra.get("max_cost_usd")
        if isinstance(max_cost_usd, (int, float)) and not isinstance(max_cost_usd, bool):
            args.extend(["--max-cost-usd", str(float(max_cost_usd))])
        per_scenario_timeout_s = extra.get("per_scenario_timeout_s")
        if isinstance(per_scenario_timeout_s, int) and per_scenario_timeout_s > 0:
            args.extend(["--per-scenario-timeout-s", str(per_scenario_timeout_s)])
        evaluator_model = extra.get("evaluator_model")
        if isinstance(evaluator_model, str) and evaluator_model.strip():
            args.extend(["--evaluator-model", evaluator_model.strip()])
        judge_model = extra.get("judge_model")
        if isinstance(judge_model, str) and judge_model.strip():
            args.extend(["--judge-model", judge_model.strip()])
        return args

    def _lifeops_bench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="lifeops_*.json")

    # multitask_bench
    def _multitask_bench_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        """Build the MultitaskBench CLI invocation.

        ``extra["harness"]`` (or ``model.model``) selects the harness: STATIC
        live harnesses ``eliza`` / ``hermes`` / ``openclaw`` need a Cerebras
        key; ``perfect`` / ``wrong`` are hermetic oracles. ``extra["lanes"]``
        (default "1,5,10") sets the concurrency levels; the N=1 baseline is
        required for the interference deltas.
        """
        harness_raw = extra.get("harness") or extra.get("agent")
        if isinstance(harness_raw, str) and harness_raw.strip():
            harness = harness_raw.strip()
        elif model.model in {"eliza", "hermes", "openclaw", "perfect", "wrong"}:
            harness = str(model.model)
        elif (model.provider or "").strip().lower() in {
            "cerebras",
            "openai",
            "groq",
            "openrouter",
            "vllm",
            "eliza",
        }:
            harness = "eliza"
        else:
            harness = "perfect"

        lanes_raw = extra.get("lanes")
        lanes = lanes_raw.strip() if isinstance(lanes_raw, str) and lanes_raw.strip() else "1,5,10"

        args = [
            python,
            "-m",
            "multitask_bench",
            "--harness",
            harness,
            "--lanes",
            lanes,
            "--output-dir",
            str(output_dir),
        ]
        model_name = extra.get("model_name") or model.model
        if isinstance(model_name, str) and model_name.strip() and model_name.strip() not in {
            "eliza",
            "hermes",
            "openclaw",
            "perfect",
            "wrong",
        }:
            args.extend(["--model", model_name.strip()])
        per_task_timeout_s = extra.get("per_task_timeout_s")
        if isinstance(per_task_timeout_s, (int, float)) and not isinstance(
            per_task_timeout_s, bool
        ):
            args.extend(["--per-task-timeout-s", str(float(per_task_timeout_s))])
        return args

    def _multitask_bench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="multitask_*.json")

    # voiceagentbench
    def _voiceagentbench_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        """Build the VoiceAgentBench CLI invocation.

        ``model.model`` selects the agent backend: ``mock`` for explicit
        hermetic smoke runs (no env vars needed); ``eliza`` / ``hermes`` /
        ``openclaw`` for cascaded-STT adapter runs. Default is ``eliza``.
        """
        agent_raw = extra.get("agent") or extra.get("harness")
        if isinstance(agent_raw, str) and agent_raw.strip():
            agent = agent_raw.strip()
        elif model.model in {"mock", "eliza", "hermes", "openclaw"}:
            agent = str(model.model)
        else:
            agent = "eliza"
        args = [
            python,
            "-m",
            "elizaos_voiceagentbench",
            "--agent",
            agent,
            "--output",
            str(output_dir),
        ]
        suite = extra.get("suite")
        if isinstance(suite, str) and suite.strip():
            args.extend(["--suite", suite.strip()])
        limit = extra.get("limit")
        if isinstance(limit, int) and limit > 0:
            args.extend(["--limit", str(limit)])
        seeds = extra.get("seeds")
        if isinstance(seeds, int) and seeds > 0:
            args.extend(["--seeds", str(seeds)])
        mock = extra.get("mock")
        if mock is True or (isinstance(mock, str) and mock.lower() == "true"):
            args.append("--mock")
        no_judge = extra.get("no_judge")
        if no_judge is True or (isinstance(no_judge, str) and no_judge.lower() == "true"):
            args.append("--no-judge")
        data_path = extra.get("data_path")
        if isinstance(data_path, str) and data_path.strip():
            args.extend(["--data-path", data_path.strip()])
        else:
            env_data_path = (
                os.environ.get("VOICEAGENTBENCH_DATA_PATH")
                or os.environ.get("VOICEAGENTBENCH_REAL_DATA_PATH")
                or ""
            ).strip()
            if env_data_path:
                args.extend(["--data-path", env_data_path])
        judge_model = extra.get("judge_model")
        if isinstance(judge_model, str) and judge_model.strip():
            args.extend(["--judge-model", judge_model.strip()])
        stt_provider = extra.get("stt_provider")
        if isinstance(stt_provider, str) and stt_provider.strip():
            args.extend(["--stt-provider", stt_provider.strip()])
        else:
            env_stt_provider = os.environ.get("VOICEAGENTBENCH_STT_PROVIDER", "").strip()
            if env_stt_provider:
                args.extend(["--stt-provider", env_stt_provider])
        _append_scenario_control_flags(args, extra)
        return args

    def _voiceagentbench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="voiceagentbench_*.json")

    def _meeting_transcription_proof_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        lane_raw = extra.get("lane")
        lane = lane_raw.strip() if isinstance(lane_raw, str) and lane_raw.strip() else ""
        if not lane:
            provider_name = (model.provider or "").strip().lower()
            lane = "mocked_plumbing" if provider_name == "mock" else "real_product"
        args = [
            python,
            "-m",
            "elizaos_meeting_transcription_proof",
            "--lane",
            lane,
            "--output",
            str(output_dir),
        ]
        manifest = extra.get("manifest")
        if isinstance(manifest, str) and manifest.strip():
            args.extend(["--manifest", manifest.strip()])
        return args

    def _meeting_voice_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        return _meeting_transcription_proof_cmd(output_dir, model, extra)

    def _meeting_voice_real_cmd(
        output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]
    ) -> list[str]:
        merged_extra = dict(extra)
        merged_extra.setdefault("lane", "real_product")
        return _meeting_transcription_proof_cmd(output_dir, model, merged_extra)

    def _meeting_transcription_proof_result(output_dir: Path) -> Path:
        return output_dir / "meeting-transcription-proof-report.json"

    # --- Hermes-native envs (tblite / terminalbench_2 / yc_bench / hermes_swe_env) ---
    # The four envs share one subprocess shim and one score extractor; only the
    # short env-id arg and the result-glob differ between them.

    hermes_run_env_cli = repo("benchmarks/hermes-adapter/run_env_cli.py")

    def _hermes_env_cmd(
        env_arg: str,
        output_dir: Path,
        model: ModelSpec,
        extra: Mapping[str, JSONValue],
    ) -> list[str]:
        args = [
            python,
            hermes_run_env_cli,
            "--env",
            env_arg,
            "--output",
            str(output_dir),
            "--model",
            (model.model or "gemma-4-31b"),
        ]
        harness = str(extra.get("agent") or extra.get("harness") or "hermes").strip().lower()
        if harness in {"eliza", "hermes", "openclaw"}:
            args.extend(["--harness", harness])
        if model.provider:
            args.extend(["--provider", model.provider])
        base_url = extra.get("base_url")
        if isinstance(base_url, str) and base_url.strip():
            args.extend(["--base-url", base_url.strip()])
        max_tasks = extra.get("max_tasks")
        if isinstance(max_tasks, int) and max_tasks > 0:
            args.extend(["--max-tasks", str(max_tasks)])
        task_filter = extra.get("task_filter")
        if isinstance(task_filter, str) and task_filter.strip():
            args.extend(["--task-filter", task_filter.strip()])
        repo_path = extra.get("repo_path")
        if isinstance(repo_path, str) and repo_path.strip():
            args.extend(["--repo-path", repo_path.strip()])
        timeout_s = extra.get("timeout_seconds")
        if isinstance(timeout_s, (int, float)) and not isinstance(timeout_s, bool) and timeout_s > 0:
            args.extend(["--timeout-seconds", str(float(timeout_s))])
        if extra.get("force") is True:
            args.append("--force")
        return args

    def _hermes_tblite_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        return _hermes_env_cmd("tblite", output_dir, model, extra)

    def _hermes_terminalbench_2_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        return _hermes_env_cmd("terminalbench_2", output_dir, model, extra)

    def _hermes_yc_bench_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        return _hermes_env_cmd("yc_bench", output_dir, model, extra)

    def _hermes_swe_env_cmd(output_dir: Path, model: ModelSpec, extra: Mapping[str, JSONValue]) -> list[str]:
        return _hermes_env_cmd("hermes_swe_env", output_dir, model, extra)

    def _hermes_tblite_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="hermes_tblite_*.json")

    def _hermes_terminalbench_2_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="hermes_terminalbench_2_*.json")

    def _hermes_yc_bench_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="hermes_yc_bench_*.json")

    def _hermes_swe_env_result(output_dir: Path) -> Path:
        return find_latest_file(output_dir, glob_pattern="hermes_hermes_swe_env_*.json")

    return [
        BenchmarkDefinition(
            id="solana",
            display_name="Solana-Gym",
            description="Solana instruction discovery benchmark (surfpool sandbox)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/solana/solana-gym-env",),
                notes=(
                    "Deterministic phase needs Bun and the bundled skill_runner dependencies. "
                    "Live LLM phase requires the selected harness/provider credentials and "
                    "Surfpool running on localhost:8899; set USE_EXTERNAL_SURFPOOL=true "
                    "to use an external Surfpool instance."
                ),
            ),
            build_command=_solana_cmd,
            locate_result=_solana_result,
            extract_score=_score_from_solana_json,
        ),
        BenchmarkDefinition(
            id="bfcl",
            display_name="BFCL",
            description="Berkeley Function-Calling Leaderboard",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes="Requires provider API key for real LLM runs; supports --provider/--model.",
            ),
            build_command=_bfcl_cmd,
            locate_result=_bfcl_result,
            extract_score=_score_from_bfcl_json,
        ),
        BenchmarkDefinition(
            id="realm",
            display_name="REALM-Bench",
            description="Real-World Planning benchmark",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("data/realm",),
                notes="Uses ./data/realm by default; set --data-path via extra config.",
            ),
            build_command=_realm_cmd,
            locate_result=_realm_result,
            extract_score=_score_from_realm_json,
        ),
        BenchmarkDefinition(
            id="mint",
            display_name="MINT",
            description="Multi-turn benchmark (tools + feedback ablations)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Runs locally through the selected harness/provider for real matrix rows; "
                    "explicit provider=mock is only for non-publishable smoke checks."
                ),
            ),
            build_command=_mint_cmd,
            locate_result=_mint_result,
            extract_score=_score_from_mint_json,
        ),
        BenchmarkDefinition(
            id="agentbench",
            display_name="AgentBench",
            description="AgentBench environments (sample tasks in this repo)",
            cwd_rel="benchmarks/agentbench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Runs sample AgentBench tasks through the selected harness/provider. "
                    "Mock runtime is only available through explicit mock configuration and "
                    "is not a real matrix score."
                ),
            ),
            build_command=_agentbench_cmd,
            locate_result=_agentbench_result,
            extract_score=_score_from_agentbench_json,
        ),
        BenchmarkDefinition(
            id="context_bench",
            display_name="ContextBench",
            description="Needle-in-a-haystack + multihop context retrieval benchmark",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Real matrix rows require the selected provider credentials. "
                    "Explicit mock mode is limited to smoke checks."
                ),
            ),
            build_command=_contextbench_cmd,
            locate_result=_contextbench_result,
            extract_score=_score_from_contextbench_json,
        ),
        BenchmarkDefinition(
            id="recall_bench",
            display_name="RecallBench",
            description="Precision/Recall/nDCG/latency over the real @elizaos/core memory-recall + knowledge-retrieval path (document-scale, per SearchMode, with a forced embed fail-open).",
            cwd_rel="packages/benchmarks/recall-bench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Secret-free and CI-safe: drives the real DocumentService / "
                    "searchMemories / FACTS path over a deterministic, committed "
                    "labeled corpus with a deterministic in-bench embedding (no "
                    "model/provider/credentials). 'tier' extra selects smoke/1k/10k."
                ),
            ),
            build_command=_recall_cmd,
            locate_result=_recall_result,
            extract_score=_score_from_recall_json,
        ),
        BenchmarkDefinition(
            id="terminal_bench",
            display_name="Terminal-Bench",
            description="Terminal proficiency benchmark",
            cwd_rel="benchmarks/terminal-bench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Real bridge/full runs require dataset/runtime setup and the selected "
                    "provider API key. Dry-run/sample modes are smoke-only."
                ),
            ),
            build_command=_terminalbench_cmd,
            locate_result=_terminalbench_result,
            extract_score=_score_from_terminalbench_json,
        ),
        BenchmarkDefinition(
            id="tau_bench",
            display_name="Tau-bench",
            description="Tool-Agent-User Interaction benchmark",
            cwd_rel="benchmarks/tau-bench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmark-data/tau-bench",),
                notes=(
                    "Real matrix rows use provider-backed LLM execution. Explicit mock mode "
                    "is available only for smoke checks."
                ),
            ),
            build_command=_tau_cmd,
            locate_result=_tau_result,
            extract_score=_score_from_taubench_json,
        ),
        BenchmarkDefinition(
            id="vending_bench",
            display_name="Vending-Bench",
            description="Vending machine management simulation benchmark",
            cwd_rel="benchmarks/vending-bench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes="Heuristic/LLM agent depending on CLI; output includes max_net_worth.",
            ),
            build_command=_vending_cmd,
            locate_result=_vending_result,
            extract_score=_score_from_vendingbench_json,
        ),
        BenchmarkDefinition(
            id="swe_bench",
            display_name="SWE-bench",
            description="Software engineering benchmark (Lite/Verified/Full)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes="Requires datasets and often Docker for evaluation; model wiring depends on runtime plugins.",
            ),
            build_command=_swe_cmd,
            locate_result=_swe_result,
            extract_score=_score_from_swebench_json,
        ),
        BenchmarkDefinition(
            id="swe_bench_orchestrated",
            display_name="SWE-bench (Orchestrated)",
            description="SWE-bench with orchestrated/direct-shell provider matrix",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Runs SWE-bench via orchestrator service or direct_shell provider path. "
                    "Supports capability contracts and full 2x3 control-plane/provider matrix."
                ),
            ),
            build_command=_swe_orchestrated_cmd,
            locate_result=_swe_orchestrated_result,
            extract_score=_score_from_swebench_orchestrated_json,
        ),
        BenchmarkDefinition(
            id="orchestrator_lifecycle",
            display_name="Orchestrator Lifecycle",
            description="Multi-turn orchestration lifecycle scenario benchmark",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/orchestrator_lifecycle/scenarios",),
                notes="Evaluates clarification, check-ins, interruptions, and stakeholder summaries.",
            ),
            build_command=_orchestrator_lifecycle_cmd,
            locate_result=_orchestrator_lifecycle_result,
            extract_score=_score_from_orchestrator_lifecycle_json,
        ),
        BenchmarkDefinition(
            id="mind2web",
            display_name="Mind2Web",
            description="Web agent navigation benchmark (OSU-NLP-Group)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Real provider runs need dataset/key setup. Sample/mock paths are "
                    "smoke-only and not publishable real matrix scores."
                ),
            ),
            build_command=_mind2web_cmd,
            locate_result=_mind2web_result,
            extract_score=_score_from_mind2web_json,
        ),
        BenchmarkDefinition(
            id="visualwebbench",
            display_name="VisualWebBench",
            description="Multimodal webpage understanding and grounding benchmark",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Real rows use the selected harness/provider. Bundled fixtures and "
                    "dry-run mode are smoke-only; --hf streams from Hugging Face when "
                    "datasets is installed."
                ),
            ),
            build_command=_visualwebbench_cmd,
            locate_result=_visualwebbench_result,
            extract_score=_score_from_visualwebbench_json,
        ),
        BenchmarkDefinition(
            id="vision_language",
            display_name="Vision-Language Bench",
            description="TextVQA, DocVQA, ChartQA, ScreenSpot, and OSWorld vision-language harness",
            cwd_rel="packages/benchmarks/vision-language",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Full runs require a real IMAGE_DESCRIPTION runtime and upstream "
                    "datasets/model assets. Explicit --smoke --stub is smoke-only and "
                    "is excluded from the real harness matrix."
                ),
            ),
            build_command=_vision_language_cmd,
            locate_result=_vision_language_result,
            extract_score=_score_from_vision_language_json,
        ),
        BenchmarkDefinition(
            id="rlm_bench",
            display_name="RLM-Bench",
            description="Recursive Language Model benchmark (S-NIAH, OOLONG) - arXiv:2512.24601",
            cwd_rel="benchmarks/rlm-bench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes="Tests long-context processing. Modes: stub (mock), rlm (full RLM). Requires RLM plugin for rlm mode.",
            ),
            build_command=_rlm_bench_cmd,
            locate_result=_rlm_bench_result,
            extract_score=_score_from_rlmbench_json,
        ),
        BenchmarkDefinition(
            id="osworld",
            display_name="OSWorld",
            description="Multimodal desktop agent benchmark (369 tasks) - arXiv:2404.07972",
            cwd_rel="benchmarks/OSWorld",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Requires VM provider: Docker (with KVM), VMware, or VirtualBox. "
                    "Uses the Eliza TypeScript benchmark bridge; dry_run smoke requires no provider key. "
                    "Set provider_name, path_to_vm (VMware), observation_type, domain, task_id via extra config."
                ),
            ),
            build_command=_osworld_cmd,
            locate_result=_osworld_result,
            extract_score=_score_from_osworld_json,
        ),
        BenchmarkDefinition(
            id="hyperliquid_bench",
            display_name="HyperliquidBench",
            description="Hyperliquid perp trading-plan generation benchmark (Eliza agent + Rust runner/evaluator)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/HyperliquidBench/dataset/domains-hl.yaml",),
                notes=(
                    "Defaults to --mode eliza (eliza TS benchmark server) with --demo "
                    "and --network testnet, so no funds are at risk. "
                    "Set agent=deterministic for the local offline smoke path. "
                    "Live network runs require building the Rust toolchain "
                    "(cd benchmarks/HyperliquidBench && cargo build --release -p hl-runner -p hl-evaluator) "
                    "AND HL_PRIVATE_KEY plus extra.no_demo=true. "
                    "Score: average final_score across scenarios (Base + Bonus − Penalty from hl-evaluator)."
                ),
            ),
            build_command=_hyperliquid_bench_cmd,
            locate_result=_hyperliquid_bench_result,
            extract_score=_score_from_hyperliquid_bench_json,
        ),
        BenchmarkDefinition(
            id="gauntlet",
            display_name="Solana Gauntlet",
            description="Tiered adversarial safety benchmark for Solana AI agents (96 scenarios, 4 levels)",
            cwd_rel="benchmarks/gauntlet",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/gauntlet/scenarios",),
                notes=(
                    "Uses ElizaOS agent with full message pipeline. "
                    "Real rows require Surfpool plus deployable or clone-backed Solana "
                    "program state; mock mode is smoke-only. Set clone_mainnet=true "
                    "only when the installed Surfpool supports the required clone path. "
                    "Scores: Task Completion (30%), Safety (40%), Efficiency (20%), Capital (10%)."
                ),
            ),
            build_command=_gauntlet_cmd,
            locate_result=_gauntlet_result,
            extract_score=_score_from_gauntlet_json,
        ),
        BenchmarkDefinition(
            id="clawbench",
            display_name="ClawBench",
            description="Deterministic scenario-based evaluation for OpenClaw agents (5 scenarios)",
            cwd_rel="benchmarks/clawbench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/clawbench/scenarios",),
                notes=(
                    "Tests tool-use decisions with fixtures (email, calendar, tasks, Slack). "
                    "Scenarios: inbox_triage, client_escalation, morning_brief, inbox_to_action, team_standup. "
                    "Scoring: safety, correctness, efficiency, structure. No LLM judge needed."
                ),
            ),
            build_command=_clawbench_cmd,
            locate_result=_clawbench_result,
            extract_score=_score_from_clawbench_json,
        ),
        BenchmarkDefinition(
            id="openclaw_bench",
            display_name="OpenClaw-Bench",
            description="AI coding assistant benchmark (setup, implementation, refactoring, testing)",
            cwd_rel="benchmarks/openclaw-benchmark",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/openclaw-benchmark/benchmark",),
                notes=(
                    "Standard tasks for AI assistants: setup (env init), implementation (weather CLI), "
                    "refactoring (modular architecture), testing (unit + integration tests). "
                    "Set task=X or all=true. Docker containers use benchmark/* naming."
                ),
            ),
            build_command=_openclaw_bench_cmd,
            locate_result=_openclaw_bench_result,
            extract_score=_score_from_openclaw_bench_json,
        ),
        BenchmarkDefinition(
            id="configbench",
            display_name="ConfigBench",
            description="Plugin configuration & secrets security benchmark (50 scripted scenarios)",
            cwd_rel="benchmarks/configbench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/configbench/src",),
                notes=(
                    "Bun runtime. Default run uses oracle/random/failing handlers (no LLM). "
                    "Set agent=eliza (or model.provider=eliza) for the LLM handler — that path "
                    "instantiates @elizaos/core in-process and needs GROQ_API_KEY or OPENAI_API_KEY."
                ),
            ),
            build_command=_configbench_cmd,
            locate_result=_configbench_result,
            extract_score=_score_from_configbench_json,
        ),
        BenchmarkDefinition(
            id="voicebench",
            display_name="VoiceBench",
            description="End-to-end voice latency benchmark (transcription + response + TTS)",
            cwd_rel="benchmarks/voicebench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/voicebench/run.sh", "benchmarks/voicebench/typescript/src/bench.ts"),
                notes=(
                    "Bun runtime via run.sh. Real profiles: groq (needs GROQ_API_KEY), "
                    "elevenlabs (needs GROQ_API_KEY and ELEVENLABS_API_KEY). Audio fixture resolved from "
                    "VOICEBENCH_AUDIO_PATH or repo defaults. "
                    "The mock profile is smoke-only and rejected by the real scorer. "
                    "Reports avg/p95/p99 end-to-end latency; lower is better."
                ),
            ),
            build_command=_voicebench_cmd,
            locate_result=_voicebench_result,
            extract_score=_score_from_voicebench_json,
        ),
        BenchmarkDefinition(
            id="mmau",
            display_name="MMAU (Audio)",
            description=(
                "Audio MMAU — Massive Multi-task Audio Understanding (Sakshi et al., "
                "ICLR 2025) — 10k audio MCQs across speech/sound/music and 27 "
                "reasoning skills. Not the Salesforce agent MMAU (arXiv:2407.18961)."
            ),
            cwd_rel="packages/benchmarks/mmau-audio",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(
                    "packages/benchmarks/mmau-audio",
                    "packages/benchmarks/mmau-audio/fixtures/smoke.jsonl",
                ),
                notes=(
                    "Pure MCQ — deterministic exact-match scoring, no LLM-judge. "
                    "Defaults to the bundled fixture + oracle agent for smoke runs; "
                    "pass extra={'hf': True} to stream from gamma-lab-umd/MMAU-test-mini "
                    "(1k) or MMAU-test (9k). Cascaded STT (Groq Whisper) discards "
                    "music/sound semantic info, so the speech category will dominate "
                    "the score until a direct-audio-input adapter lands."
                ),
            ),
            build_command=_mmau_cmd,
            locate_result=_mmau_result,
            extract_score=_score_from_mmau_json,
        ),
        BenchmarkDefinition(
            id="voicebench_quality",
            display_name="VoiceBench (quality)",
            description=(
                "Vendored VoiceBench (Chen et al. 2024) — 8-suite quality "
                "benchmark over 6783 spoken instructions"
            ),
            cwd_rel="packages/benchmarks/voicebench-quality",
            requirements=BenchmarkRequirements(
                env_vars=("CEREBRAS_API_KEY",),
                paths=("packages/benchmarks/voicebench-quality/elizaos_voicebench",),
                notes=(
                    "Cascaded STT (Groq Whisper or local Eliza runtime) → text adapter "
                    "(eliza/hermes/openclaw). "
                    "Judged suites scored by gemma-4-31b on Cerebras. "
                    "Score is the unweighted mean of per-suite scores in [0, 1]; "
                    "real runs require a real STT provider: GROQ_API_KEY for groq, "
                    "VOICEBENCH_QUALITY_STT_PROVIDER=eliza-runtime with ELIZA_API_BASE/"
                    "ELIZA_BENCH_URL, or VOICEBENCH_QUALITY_STT_PROVIDER=faster-whisper. "
                    "higher is better. extra.mock=true/provider=mock is smoke-only "
                    "and rejected by the real scorer."
                ),
            ),
            build_command=_voicebench_quality_cmd,
            locate_result=_voicebench_quality_result,
            extract_score=_score_from_voicebench_quality_json,
        ),
        BenchmarkDefinition(
            id="social_alpha",
            display_name="Social-Alpha",
            description="Trust marketplace benchmark on real Discord crypto-chat data (EXTRACT/RANK/DETECT/PROFIT)",
            cwd_rel="benchmarks/social-alpha",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/social-alpha/fixtures/smoke-data",),
                notes=(
                    "Defaults to the rule-based BaselineSystem (no LLM). Set system=eliza|full|smart|oracle "
                    "via extra to swap implementations; eliza/full additionally need provider keys. "
                    "Uses the bundled smoke fixture when the full dataset is absent. "
                    "Score: composite Trust Marketplace Score (0..1)."
                ),
            ),
            build_command=_social_alpha_cmd,
            locate_result=_social_alpha_result,
            extract_score=_score_from_social_alpha_json,
        ),
        BenchmarkDefinition(
            id="trust",
            display_name="Trust",
            description="Agent trust/security detection benchmark",
            cwd_rel="benchmarks/trust",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/trust/elizaos_trust_bench",),
                notes=(
                    "Defaults to the oracle handler for deterministic no-key smoke runs. "
                    "Set handler=random for a baseline or handler=eliza/handler=eliza-bridge "
                    "for agent-backed runs; those paths require their runtime/provider setup."
                ),
            ),
            build_command=_trust_cmd,
            locate_result=_trust_result,
            extract_score=_score_from_trust_json,
        ),
        BenchmarkDefinition(
            id="webshop",
            display_name="WebShop",
            description="WebShop product-search/purchase benchmark with Eliza agent",
            cwd_rel="benchmarks/webshop",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/webshop/elizaos_webshop",),
                notes=(
                    "Defaults to bundled sample tasks (--sample). Set hf=true to load from HuggingFace. "
                    "Real-LLM mode is the default and needs a provider key (GROQ/OPENAI/etc.); "
                    "set mock=true for a deterministic smoke run. Score: success_rate."
                ),
            ),
            build_command=_webshop_cmd,
            locate_result=_webshop_result,
            extract_score=_score_from_webshop_json,
        ),
        BenchmarkDefinition(
            id="woobench",
            display_name="WooBench",
            description="Mystical reading conversation and revenue benchmark",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("benchmarks/woobench",),
                notes=(
                    "Default run uses the eliza TS benchmark bridge plus the LLM evaluator. "
                    "Set mock=true or agent=dummy with evaluator=heuristic for a deterministic "
                    "no-credential smoke run. Score is overall_score normalized from 0..100 to 0..1."
                ),
            ),
            build_command=_woobench_cmd,
            locate_result=_woobench_result,
            extract_score=_score_from_woobench_json,
        ),
        BenchmarkDefinition(
            id="scambench",
            display_name="ScamBench",
            description="Adversarial scam-detection benchmark (refusal vs helpfulness)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(
                    "../../training/data/normalized/scambench.jsonl",
                    "../../training/data/synthesized/scambench/scambench.jsonl",
                ),
                notes=(
                    "Reads the normalized + Claude-teacher-labeled scambench dataset. "
                    "Score is the equally-weighted mean of refusal-correctness on scam prompts "
                    "and helpfulness on legit prompts. Higher better."
                ),
            ),
            build_command=_scambench_cmd,
            locate_result=_scambench_result,
            extract_score=_score_from_scambench_json,
        ),
        BenchmarkDefinition(
            id="abliteration-robustness",
            display_name="Abliteration Robustness",
            description="Over-refusal benchmark for abliterated model variants on benign prompts",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Loads the harmless prompt set used by training/scripts/training/abliterate.py "
                    "(default HF dataset mlabonne/harmless_alpaca). Score = 1 - refusal_rate. "
                    "Pair with the abliterated variant to assert it still helps on benign requests."
                ),
            ),
            build_command=_abliteration_robustness_cmd,
            locate_result=_abliteration_robustness_result,
            extract_score=_score_from_abliteration_robustness_json,
        ),
        BenchmarkDefinition(
            id="action-calling",
            display_name="Action Calling",
            description="Native function/tool calling against planner-style records",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("training/data/native/records/hermes-fc-v1.jsonl",),
                notes=(
                    "Samples native planner records and sends OpenAI-compatible tools to the provider. "
                    "Asserts real tool-call emission, tool-name match, args JSON parse, required-arg presence, "
                    "and expected argument-value preservation. Score = geometric mean of the five sub-rates."
                ),
            ),
            build_command=_action_calling_cmd,
            locate_result=_action_calling_result,
            extract_score=_score_from_action_calling_json,
        ),
        BenchmarkDefinition(
            id="lifeops_bench",
            display_name="LifeOpsBench",
            description="Multi-turn life-assistant tool-use benchmark (calendar/mail/messages/contacts/reminders/finance/travel/health/sleep/focus)",
            cwd_rel="packages/benchmarks/lifeops-bench",
            requirements=BenchmarkRequirements(
                env_vars=("CEREBRAS_API_KEY", "ANTHROPIC_API_KEY"),
                paths=(
                    "packages/benchmarks/lifeops-bench/eliza_lifeops_bench",
                    "packages/benchmarks/lifeops-bench/data/snapshots",
                ),
                notes=(
                    "model.model selects the agent backend: 'perfect'/'wrong' for hermetic oracle runs (no env vars needed); "
                    "'hermes'/'cerebras-direct'/'eliza' for live adapters. "
                    "CEREBRAS_API_KEY is required when LIVE scenarios are scheduled (simulated user uses gemma-4-31b). "
                    "ANTHROPIC_API_KEY is required for the LIVE judge (claude-opus-4-7). "
                    "Cost cap defaults to $10; override via extra.max_cost_usd. "
                    "Score: pass@1 across all (scenario, seed) pairs. Higher is better."
                ),
            ),
            build_command=_lifeops_bench_cmd,
            locate_result=_lifeops_bench_result,
            extract_score=_score_from_lifeops_bench_json,
        ),
        BenchmarkDefinition(
            id="multitask_bench",
            display_name="MultitaskBench",
            description="One agent handling N interleaved LifeOps tasks (N=1/5/10); per-task score interference under load for eliza/hermes/openclaw",
            cwd_rel="packages/benchmarks/multitask-bench",
            requirements=BenchmarkRequirements(
                env_vars=("CEREBRAS_API_KEY",),
                paths=(
                    "packages/benchmarks/multitask-bench/multitask_bench",
                    "packages/benchmarks/lifeops-bench/eliza_lifeops_bench",
                    "packages/benchmarks/lifeops-bench/data/snapshots",
                ),
                notes=(
                    "extra.harness (or model.model) selects the harness: 'perfect'/'wrong' for hermetic oracle runs (no env vars needed); "
                    "'eliza'/'hermes'/'openclaw' for live adapters. "
                    "STATIC-only sample (10 scenarios) — no ANTHROPIC key or LIVE judge needed. "
                    "CEREBRAS_API_KEY is required for the live harnesses (agent model gemma-4-31b). "
                    "The eliza live lane also needs the per-session usage-buffer fix in packages/lifeops-bench/src/server.ts (issue #13777 PR 1); "
                    "set MULTITASK_ELIZA_USAGE_FIX=1 once it is in tree. "
                    "extra.lanes (default '1,5,10') sets concurrency; N=1 is the interference baseline and is required. "
                    "isolation is 'shared_runtime' for eliza, 'process_per_turn' for hermes/openclaw. "
                    "Score: mean_task_score of the N=10 lane. Higher is better."
                ),
            ),
            build_command=_multitask_bench_cmd,
            locate_result=_multitask_bench_result,
            extract_score=_score_from_multitask_bench_json,
        ),
        BenchmarkDefinition(
            id="voiceagentbench",
            display_name="VoiceAgentBench",
            description="Voice-in + tool-call-out + multi-turn benchmark (single/parallel/sequential/multi-turn/safety/multilingual)",
            cwd_rel="packages/benchmarks/voiceagentbench",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(
                    "packages/benchmarks/voiceagentbench/elizaos_voiceagentbench",
                    "packages/benchmarks/voiceagentbench/fixtures",
                ),
                notes=(
                    "model.model selects the agent backend: 'mock' for smoke-only fixture runs; "
                    "'eliza'/'hermes'/'openclaw' for real cascaded-STT adapter runs. "
                    "Real runs require upstream Hugging Face audio (or audio_b64 records) plus "
                    "GROQ_API_KEY for Whisper, VOICEAGENTBENCH_STT_PROVIDER=eliza-runtime with "
                    "ELIZA_API_BASE/ELIZA_BENCH_URL, or VOICEAGENTBENCH_STT_PROVIDER=faster-whisper. "
                    "CEREBRAS_API_KEY is required for the multi-turn coherence judge (gemma-4-31b). "
                    "Set extra.mock=true and extra.suite=single|parallel|sequential|multi-turn|safety|multilingual|all. "
                    "Score: pass@1 across all (task, seed) pairs. Higher is better."
                ),
            ),
            build_command=_voiceagentbench_cmd,
            locate_result=_voiceagentbench_result,
            extract_score=_score_from_voiceagentbench_json,
        ),
        BenchmarkDefinition(
            id="meeting_voice",
            display_name="Meeting Voice Smoke",
            description=(
                "No-key smoke lane for meeting voice transcription proof wiring, "
                "canonical artifact shape, capture-path metadata, and evidence "
                "bundle validation. Mocked plumbing is never product proof."
            ),
            cwd_rel="packages/benchmarks/meeting-transcription-proof",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(
                    "packages/benchmarks/meeting-transcription-proof/elizaos_meeting_transcription_proof",
                    "packages/benchmarks/meeting-transcription-proof/fixtures/mock-meeting-manifest.json",
                ),
                notes=(
                    "Alias for meeting_transcription_proof. With a mock provider it builds "
                    "the lane='mocked_plumbing' no-key smoke route; with a real provider it "
                    "builds lane='real_product'. Use meeting_voice_real, meeting_voice_stress, "
                    "or meeting_voice_av with extra.manifest=<path> for reviewed product evidence."
                ),
            ),
            build_command=_meeting_voice_cmd,
            locate_result=_meeting_transcription_proof_result,
            extract_score=_score_from_meeting_transcription_proof_json,
        ),
        BenchmarkDefinition(
            id="meeting_voice_real",
            display_name="Meeting Voice Real Product Evidence",
            description=(
                "Manual real-product lane for Zoom, Google Meet, on-device, cloud-agent, "
                "and hybrid local/cloud meeting transcription proof"
            ),
            cwd_rel="packages/benchmarks/meeting-transcription-proof",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("packages/benchmarks/meeting-transcription-proof/elizaos_meeting_transcription_proof",),
                notes=(
                    "Manual evidence-gated alias for meeting_transcription_proof with "
                    "lane='real_product'. Requires extra.manifest=<path> containing real "
                    "media/log/screenshot/model artifact evidence; mock provider runs fail closed."
                ),
            ),
            build_command=_meeting_voice_real_cmd,
            locate_result=_meeting_transcription_proof_result,
            extract_score=_score_from_meeting_transcription_proof_json,
        ),
        BenchmarkDefinition(
            id="meeting_voice_stress",
            display_name="Meeting Voice Acoustic Stress Evidence",
            description=(
                "Manual real-product lane for meeting voice stressors: music, noise, "
                "babble, overlap, far-field room audio, and multi-speaker single-stream cases"
            ),
            cwd_rel="packages/benchmarks/meeting-transcription-proof",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("packages/benchmarks/meeting-transcription-proof/elizaos_meeting_transcription_proof",),
                notes=(
                    "Manual evidence-gated alias for meeting_transcription_proof with "
                    "lane='real_product'. The manifest must include stress dataset sources, "
                    "metrics, media refs, logs, and reviewed model outputs."
                ),
            ),
            build_command=_meeting_voice_real_cmd,
            locate_result=_meeting_transcription_proof_result,
            extract_score=_score_from_meeting_transcription_proof_json,
        ),
        BenchmarkDefinition(
            id="meeting_voice_av",
            display_name="Meeting Voice Audio-Visual Evidence",
            description=(
                "Manual real-product lane for audio-visual meeting proof, active-speaker "
                "metadata, video evidence, screenshots, and transcript/diarization artifacts"
            ),
            cwd_rel="packages/benchmarks/meeting-transcription-proof",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("packages/benchmarks/meeting-transcription-proof/elizaos_meeting_transcription_proof",),
                notes=(
                    "Manual evidence-gated alias for meeting_transcription_proof with "
                    "lane='real_product'. Requires an AV manifest with video, active speaker, "
                    "media, logs, screenshots, and manual review artifacts."
                ),
            ),
            build_command=_meeting_voice_real_cmd,
            locate_result=_meeting_transcription_proof_result,
            extract_score=_score_from_meeting_transcription_proof_json,
        ),
        BenchmarkDefinition(
            id="meeting_transcription_proof",
            display_name="Meeting Transcription Proof",
            description=(
                "Issue #12486 proof registry for Zoom, Google Meet, on-device "
                "capture, cloud agents, hybrid inference, diarization, speaker "
                "identity, consent, retention, and evidence bundles"
            ),
            cwd_rel="packages/benchmarks/meeting-transcription-proof",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(
                    "packages/benchmarks/meeting-transcription-proof/elizaos_meeting_transcription_proof",
                    "packages/benchmarks/meeting-transcription-proof/fixtures/mock-meeting-manifest.json",
                ),
                notes=(
                    "Set extra.lane='mocked_plumbing' for the no-key schema/capture/evidence "
                    "fixture lane, or extra.lane='real_product' plus extra.manifest=<path> "
                    "for real Zoom/Google Meet/on-device/cloud/hybrid evidence. The scorer "
                    "marks mocked reports non-publishable and requires complete evidence files "
                    "for real_product reports."
                ),
            ),
            build_command=_meeting_transcription_proof_cmd,
            locate_result=_meeting_transcription_proof_result,
            extract_score=_score_from_meeting_transcription_proof_json,
        ),
        # ----- standard public LLM benchmarks (W1-B1, gap C6) -----
        BenchmarkDefinition(
            id="mmlu",
            display_name="MMLU",
            description="Massive Multitask Language Understanding (cais/mmlu, 4-way multiple choice over 57 subjects)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Talks to any OpenAI-compatible chat-completion endpoint via "
                    "--model-endpoint or extra.model_endpoint. Mock provider uses "
                    "a bundled fixture (no network, no HF datasets install). Real "
                    "runs require `datasets` and pull cais/mmlu lazily."
                ),
            ),
            build_command=_mmlu_cmd,
            locate_result=_mmlu_result,
            extract_score=_score_from_mmlu_json,
        ),
        BenchmarkDefinition(
            id="humaneval",
            display_name="HumanEval",
            description="OpenAI HumanEval pass@1 over openai_humaneval (164 Python coding problems)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Talks to any OpenAI-compatible chat-completion endpoint. "
                    "Each completion is exec'd in a sandboxed subprocess with a "
                    "per-test timeout. Mock provider uses a bundled fixture. "
                    "Use extra.timeout_s to override the default 10s test timeout."
                ),
            ),
            build_command=_humaneval_cmd,
            locate_result=_humaneval_result,
            extract_score=_score_from_humaneval_json,
        ),
        BenchmarkDefinition(
            id="gsm8k",
            display_name="GSM8K",
            description="Grade-school math word problems (openai/gsm8k) with strict #### integer parsing",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Talks to any OpenAI-compatible chat-completion endpoint. "
                    "Prompts for chain-of-thought ending in '#### <integer>'. "
                    "Mock provider uses a bundled fixture (no network)."
                ),
            ),
            build_command=_gsm8k_cmd,
            locate_result=_gsm8k_result,
            extract_score=_score_from_gsm8k_json,
        ),
        BenchmarkDefinition(
            id="mt_bench",
            display_name="MT-Bench",
            description="Multi-turn open-ended LLM benchmark judged 1-10 by a strong model (LMSYS-style)",
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=(),
                notes=(
                    "Candidate model and judge model both talk to OpenAI-compatible "
                    "endpoints. Use extra.judge_endpoint + extra.judge_model + "
                    "extra.judge_api_key_env to point the judge at a separate "
                    "strong model (gpt-4o, claude-opus, eliza-1-70b, etc). "
                    "Score = mean 1-10 judge rating divided by 10."
                ),
            ),
            build_command=_mt_bench_cmd,
            locate_result=_mt_bench_result,
            extract_score=_score_from_mt_bench_json,
        ),
        BenchmarkDefinition(
            id="trajectory_replay",
            display_name="Trajectory Replay",
            description=(
                "Regression benchmark that replays curated eliza_native_v1 "
                "trajectories from ~/.eliza/trajectories against a candidate "
                "endpoint and scores action-sequence + final-state match via "
                "eliza_reward_fn (closes M5 follow-up)."
            ),
            cwd_rel=".",
            requirements=BenchmarkRequirements(
                env_vars=(),
                paths=("packages/training/scripts/eliza_reward_fn.py",),
                notes=(
                    "Required extras: traj_set (directory of trajectory JSON "
                    "files) and baseline (baseline model id whose recorded "
                    "outputs are the ground truth). Optional knobs: "
                    "reward_threshold (default 0.5), exact_action_sequence "
                    "(default True), action_weight + final_state_weight "
                    "(default 0.5/0.5), max_tokens. Score is the mean per-"
                    "trajectory aggregate in [0, 1]; higher is better."
                ),
            ),
            build_command=_trajectory_replay_cmd,
            locate_result=_trajectory_replay_result,
            extract_score=_score_from_trajectory_replay_json,
        ),
        BenchmarkDefinition(
            id="hermes_tblite",
            display_name="Hermes TBlite",
            description=(
                "Hermes-agent's TBlite environment (100 calibrated terminal tasks). "
                "Fastest of the four hermes-native envs — preferred for smoke loops."
            ),
            cwd_rel="benchmarks/hermes-adapter",
            requirements=BenchmarkRequirements(
                env_vars=("CEREBRAS_API_KEY",),
                paths=("benchmarks/hermes-adapter",),
                notes=(
                    "Runs hermes-agent's tblite_env evaluate flow via run_env_cli.py. "
                    "Defaults are smoke-friendly (max_tasks=5). Override via extra: "
                    "max_tasks, task_filter, base_url, repo_path, force, timeout_seconds."
                ),
            ),
            build_command=_hermes_tblite_cmd,
            locate_result=_hermes_tblite_result,
            extract_score=_score_from_hermes_env_json,
        ),
        BenchmarkDefinition(
            id="hermes_terminalbench_2",
            display_name="Hermes TerminalBench 2",
            description="Hermes-agent's terminalbench_2 environment (89 terminal tasks).",
            cwd_rel="benchmarks/hermes-adapter",
            requirements=BenchmarkRequirements(
                env_vars=("CEREBRAS_API_KEY",),
                paths=("benchmarks/hermes-adapter",),
                notes=(
                    "Runs hermes-agent's terminalbench_2 env via run_env_cli.py. "
                    "Same extra-config knobs as hermes_tblite."
                ),
            ),
            build_command=_hermes_terminalbench_2_cmd,
            locate_result=_hermes_terminalbench_2_result,
            extract_score=_score_from_hermes_env_json,
        ),
        BenchmarkDefinition(
            id="hermes_yc_bench",
            display_name="Hermes YC-Bench",
            description="Hermes-agent's yc_bench environment (long-horizon strategic tasks).",
            cwd_rel="benchmarks/hermes-adapter",
            requirements=BenchmarkRequirements(
                env_vars=("CEREBRAS_API_KEY",),
                paths=("benchmarks/hermes-adapter",),
                notes=(
                    "Runs hermes-agent's yc_bench env via run_env_cli.py. "
                    "Long-horizon — set max_tasks low for smoke runs."
                ),
            ),
            build_command=_hermes_yc_bench_cmd,
            locate_result=_hermes_yc_bench_result,
            extract_score=_score_from_hermes_env_json,
        ),
        BenchmarkDefinition(
            id="hermes_swe_env",
            display_name="Hermes SWE Env",
            description="Hermes-agent's SWE-bench-style hermes_swe_env environment.",
            cwd_rel="benchmarks/hermes-adapter",
            requirements=BenchmarkRequirements(
                env_vars=("CEREBRAS_API_KEY",),
                paths=("benchmarks/hermes-adapter",),
                notes=(
                    "Runs hermes-agent's hermes_swe_env evaluate flow via run_env_cli.py. "
                    "SWE-bench style; expect long per-task runtime."
                ),
            ),
            build_command=_hermes_swe_env_cmd,
            locate_result=_hermes_swe_env_result,
            extract_score=_score_from_hermes_env_json,
        ),
    ]


def load_benchmark_result_json(path: Path) -> JSONValue:
    return load_json_file(path)