tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_v1.yml

# Cloud-OpsBench v1 — opensre+LLM vs the paper's published LLM-alone baselines.
#
# This config pins specific provider snapshots so the run is reproducible.
# Pre-registration at preregistrations/cloudopsbench_v1.yml MUST have its
# YAUHEN: placeholders filled in and be committed to git BEFORE running
# without --dev. (The framework refuses production runs without a real
# pre-registration; see IntegrityGuard.pre_flight, Mechanism 1.)
#
# Run for the first time with --dev to verify the chain and see real
# non-zero scores against the paper's models. Drop --dev once the
# pre-registration is committed and you're publishing numbers:
#
#   uv run python -m tests.benchmarks._framework.cli validate \
#       tests/benchmarks/cloudopsbench/configs/cloudopsbench_v1.yml
#
#   uv run python -m tests.benchmarks._framework.cli run \
#       tests/benchmarks/cloudopsbench/configs/cloudopsbench_v1.yml --dev   # first run
#   uv run python -m tests.benchmarks._framework.cli run \
#       tests/benchmarks/cloudopsbench/configs/cloudopsbench_v1.yml         # production

benchmark: cloudopsbench

modes:
  - opensre+llm

# Four LLMs the paper benchmarks. Each one has its own LLMSpec in
# tests/benchmarks/_framework/llm_dispatch.py — the runner activates them
# one at a time (serializes across LLMs, parallelizes within an LLM).
llms:
  - claude-4-sonnet
  - gpt-4o
  - gpt-5
  - deepseek-v3.2

# Pinned model versions — the framework refuses to start the run if any
# model resolves to a different snapshot. Honest reproducibility: the
# same config + same SHA should always hit the same model bytes.
model_versions:
  claude-4-sonnet: claude-sonnet-4-5-20250929
  gpt-4o: gpt-4o-2024-11-20
  gpt-5: gpt-5-2025-08-07
  deepseek-v3.2: deepseek-chat-v3.2

# Replication — required ≥3 for stochastic LLMs (Box-Hunter-Hunter Ch 3.4).
# Variance estimate is meaningless below 3.
runs_per_case: 3

# Parallelism within a single LLM activation. The runner serializes across
# LLMs (no two providers active at once) but parallelizes within. On
# Fargate with 2 vCPU, 4 workers is the right balance.
workers: 4

# Hard cost cap — framework calls CostBudgetExceeded the moment the next
# API call would push us over, and halts the run with a partial-completion
# report. Set conservatively; bump for the full 452-case grid (estimate:
# ~$200-500 across all 4 models × 452 cases × 3 runs at current pricing).
cost_budget_usd: 500.0

# Seeded random case selection — Mechanism 6 (no cherry-picking). The same
# seed always picks the same case set, so a re-run is comparable.
seed: 42

# Where artifacts land. The Fargate entrypoint syncs this whole tree to
# s3://$BENCH_RESULTS_BUCKET/runs/ when the bench finishes.
output_dir: .bench-results/cloudopsbench_v1/

# Pre-registration — Mechanism 1. The framework's IntegrityGuard.pre_flight
# refuses production runs without a real, non-empty pre-registration.
# Fill the YAUHEN: placeholders in this file before dropping --dev.
pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

# Filters — start with a tractable subset for the first real-LLM run, then
# widen for the full publication grid. The defaults below pull a 30-case
# sample of seen-shape (Startup + Runtime) cases — enough to see whether
# the paper's per-model baseline reproduces (A@1 > 0.65 universal on seen
# shapes per Cloud-OpsBench Table 4).
#
# For the full grid: comment out `limit` and `seen_shape` filters entirely.
filters:
  limit: 30
  seen_shape: [true]
  systems: []
  fault_categories: []

report_formats:
  - json
  - markdown
  - html