tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_v1_openai.yml

# Cloud-OpsBench v1 — OpenAI slice (gpt-4o + gpt-5).
#
# Provider slice of cloudopsbench_v1.yml. Use this when running just the
# OpenAI models (no ANTHROPIC_API_KEY / DEEPSEEK_API_KEY required). The
# full grid (all 4 paper models in one run) is still
# cloudopsbench_v1.yml — keep that for the publication-grade comparison
# once all provider credits are available.
#
# Sibling slices:
#   - cloudopsbench_v1_anthropic.yml   (claude-4-sonnet)
#   - cloudopsbench_v1_deepseek.yml    (deepseek-v3.2)
#
# Required env at run time: OPENAI_API_KEY.
#
# Run with --dev first to verify the chain, then drop --dev for production.
# Pin the MIN_TOOL_CALLS floor explicitly so it lands in provenance.json:
#   BENCH_MIN_TOOL_CALLS=5 uv run python -m tests.benchmarks._framework.cli run \
#       tests/benchmarks/cloudopsbench/configs/cloudopsbench_v1_openai.yml --dev
#
# ===========================================================================
# 2026-06-07 RE-RUN NOTES (read before launching the powered run)
# ---------------------------------------------------------------------------
# 1. VOCAB FIX (predictor._ROOT_CAUSES): the 2026-06-06 run scored ~0.01 a1 on
#    the entire unseen-shape stratum (performance + admission) because 7
#    root-cause tokens were missing from the predictor vocabulary — a scorer
#    artifact, not a model failure (object_a1 was ~0.40 there). Those tokens
#    are now in the vocab, so this re-run is the FIRST whose unseen-shape
#    numbers are trustworthy. Validate cheaply first via
#    cloudopsbench_vocabpilot_openai.yml before spending on the full grid.
#
# 2. FARGATE WALL-TIME (the 2026-06-06 run aborted at ~9h / 28% complete).
#    Observed throughput was ~14 s/cell at workers=1. The full grid is 8136
#    cells, so:
#        workers=1 ⇒ ~32h    workers=2 ⇒ ~16h    workers=4 ⇒ ~8h
#    Whatever killed the task at ~9h (ECS task timeout / OOM / spot reclaim)
#    WILL repeat at workers=1. Before launching, do ONE of:
#      (a) Confirm the ECS stop reason and raise the task timeout / move off
#          spot so a 32h task can finish, OR
#      (b) Upgrade to OpenAI tier-2 (450k TPM) and set workers=4 to land near
#          ~8h — but DO NOT set workers>1 on tier-1 (30k TPM): the June-3 run
#          showed a rate-limit storm that crashed 78% of cells (a1 → 2.8%), OR
#      (c) CHUNK the grid into independent sub-runs each well under the wall
#          and merge later. Outputs are joinable by case_id + llm, e.g. split
#          by llm and shape:
#            BENCH_MIN_TOOL_CALLS=5 ... run <this config copy: gpt-4o, seen>
#            BENCH_MIN_TOOL_CALLS=5 ... run <this config copy: gpt-4o, unseen>
#            BENCH_MIN_TOOL_CALLS=5 ... run <this config copy: gpt-5,  seen>
#            BENCH_MIN_TOOL_CALLS=5 ... run <this config copy: gpt-5,  unseen>
#          Each chunk is ~2000 cells ≈ ~8h at workers=1 — survivable.
#    Option (c) is the lowest-risk path that needs no infra/tier change.
# ===========================================================================

benchmark: cloudopsbench

# Three-arm contrast (locks the audit-grade attribution):
#   opensre+llm        full opensre   prompt + MIN_TOOL_CALLS floor (=5, via env)
#   llm_alone          same prompt    no MIN_TOOL_CALLS floor
#   llm_alone_pure     minimal prompt no MIN_TOOL_CALLS floor
#
# The floor is read from BENCH_MIN_TOOL_CALLS at import time (default 5) and
# stamped into provenance.json under run_inputs.min_tool_calls. Set it
# explicitly on the command line so the run records the value you intended.
#
# Reading the contrasts (cost: +50% per arm added):
#   (opensre+llm) - (llm_alone)      = lift from the MIN_TOOL_CALLS floor alone
#   (opensre+llm) - (llm_alone_pure) = lift from opensre's full stack
#   (llm_alone)   - (llm_alone_pure) = lift from opensre's prompt alone
modes:
  - opensre+llm
  - llm_alone
  - llm_alone_pure

llms:
  - gpt-4o
  - gpt-5

model_versions:
  gpt-4o: gpt-4o-2024-11-20
  gpt-5: gpt-5-2025-08-07

runs_per_case: 3

# Workers ↔ provider tier math:
#   - gpt-4o tier-1 cap: 30k TPM
#   - Each CloudOpsBench cell burns ~25-30k tokens per investigation
#   - workers=4 ⇒ ~120k/min demand vs 30k available ⇒ rate-limit storm
#     (June-3 run: 78% of cells died with "rate-limited" error,
#      a1 mean fell to 2.8%)
#   - workers=1 ⇒ one investigation at a time; retries drain the bucket
#     between cells; ~90%+ cell-completion rate.
# Wall-time: ~3-4× longer than workers=4, but the cells actually complete.
# After upgrading to OpenAI tier-2 (450k TPM gpt-4o), can return to 4.
workers: 1

cost_budget_usd: 500.0
seed: 42

# Per-slice output dir so artifacts from different provider runs don't
# stomp on each other under .bench-results/. Each report.json is kept
# alongside its own provenance.json — joinable later by case_id + llm.
output_dir: .bench-results/cloudopsbench_v1_openai/

# Same pre-registration as the full grid — slices share the experimental
# scope; only the LLM set differs. IntegrityGuard.pre_flight still applies
# to non-dev runs.
pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

# Power the sample for an audited bench run.
#
# 06-05 11:46 run on `limit: 30, seen_shape: [true]` produced a 95% CI half-width
# of ±0.13 on a1 — wider than the effect we're trying to resolve vs the paper
# baselines. At the full corpus (452 cases) and both shape strata, the
# scenario-clustered bootstrap CI tightens to ~±0.05, enough to make a
# "matches paper / beats paper" claim statistically defensible.
#
# Also unblocks the overfit guard — at 100% seen-shape + 100% non-held-out the
# all / seen-shape / optimize strata in the report came out identical, so the
# per-stratum mechanism couldn't surface any signal. Both shape strata + the
# 20% held-out split (from preregistrations/cloudopsbench_v1.yml) are required
# for the generalization_gate computation in the report to mean anything.
#
# Cost projection (vs 11:46's $1.67 at 180 cells, gpt-5 dispatched as gpt-4o):
#   - With ALL three modes and full corpus:
#     8136 cells = 452 cases × 2 LLMs × 3 seeds × 3 modes
#   - With the dispatcher fix, gpt-5 actually runs gpt-5 (~5× per-call cost
#     vs gpt-4o). With llm_alone* lower MIN_TOOL_CALLS, baseline cells average
#     fewer tool calls per case, partially offsetting the tripled cell count.
#   - Expected total: ~$120. Well within the $500 budget but ~70× the
#     11:46 spend. Confirm budget before triggering.
#   - To run cheaper:
#     - Drop llm_alone_pure → saves ~$40 (loses the prompt-vs-floor attribution)
#     - Drop both baselines → saves ~$80 (loses the internal contrast entirely;
#       reverts to "compare against paper's number from a different harness",
#       which the prereg's comparison_protocol forbids for the primary claim)
filters:
  # limit: omit → run all 452 cases
  seen_shape: [true, false]
  systems: []
  fault_categories: []

report_formats:
  - json
  - markdown
  - html