tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_definitive_openai.yml

# Cloud-OpsBench — definitive opensre+llm vs pure-LLM comparison (gpt-4o).
#
# Single goal: produce a tight, statistically defensible answer to
# "does opensre's structural investigation beat a minimal-prompt LLM
#  using the same tools, predictor, and scoring?"
#
# Why this config (and not the others):
#   - 2 arms (opensre+llm + llm_alone_pure): the comparison Yauhen wants
#     certainty on. llm_alone (full opensre prompt, no floor) is omitted —
#     it sits between the two and is less informative for the headline
#     question.
#   - gpt-4o only: the published paper baseline (0.49 aggregate) is on
#     this model. gpt-5 would double cost without changing the
#     opensre-vs-pure-LLM verdict.
#   - Full 452 cases × 3 seeds = 2,712 cells. n=452 scenarios gives the
#     paired-bootstrap CI a half-width of ~±0.025 on the contrast, enough
#     to definitively accept or reject "opensre measurably wins / loses."
#
# What the 2026-06-06 partial (n=254, OLD code, 28% coverage) showed:
#   (opensre+llm) - (llm_alone_pure) = -0.025 [-0.054, +0.004]
#   Point estimate negative; CI included 0 narrowly. The full grid here
#   tightens that CI by ~3x at n=452.
#
# Pre-registered expectation (write the prediction BEFORE seeing the run):
#   - Point estimate: weakly negative on aggregate (opensre at -0.01 to
#     -0.03 vs pure-LLM)
#   - Seen-shape: pure-LLM ahead by ~0.05-0.10 (matches Fargate Fix-A
#     validation showing -0.097 at n=24)
#   - Unseen-shape: closer to even (matches Anthropic pilot where vocab
#     fix dominated and prompt structure mattered less)
#   - If aggregate Δa1 > 0: report as a win that the partial missed.
#   - If aggregate Δa1 ≈ 0 ± noise: report as honest equivalence.
#   - If aggregate Δa1 < 0 (excludes 0): report the negative result
#     honestly. Don't iterate further on cloudopsbench.
#
# Run (NEEDS Anthropic creds top up — Fargate predictor uses gpt-4o but
# the agent also makes openai calls; previous Fargate runs died at credit
# exhaustion):
#   cd opensre && set -a && source .env && set +a
#   BENCH_MIN_TOOL_CALLS=5 uv run python -m tests.benchmarks._framework.cli \
#     run tests/benchmarks/cloudopsbench/configs/cloudopsbench_definitive_openai.yml --dev
#
# Wall-time risk: prior 06-06 Fargate run died at 9h from OpenAI credit
# exhaustion. workers=4 should finish in ~5-6h; budget headroom in the
# OpenAI account for ~$80 + 30% margin = $110 before launching.

benchmark: cloudopsbench

modes:
  - opensre+llm
  - llm_alone_pure

llms:
  - gpt-4o

model_versions:
  gpt-4o: gpt-4o-2024-11-20

# 3 runs/case averages single-shot LLM noise on stochastic completions.
runs_per_case: 3

# workers=4: OpenAI tier-2 confirmed sufficient. Wall time ~5-6h.
# Drop to workers=2 if you suspect rate limits; ~10-11h.
workers: 4

# $100 covers projected $80 + 25% headroom.
cost_budget_usd: 100.0
seed: 42

output_dir: .bench-results/cloudopsbench_definitive_openai/

pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

filters:
  # Full corpus, both shape strata — no limit.
  seen_shape: [true, false]
  systems: []
  fault_categories: []

report_formats:
  - json
  - markdown
  - html