tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_fixa_validation_openai.yml

# Cloud-OpsBench — Fix-A + vocab-fix validation (cheap, gpt-4o, two arms).
#
# Validates two 2026-06-07 fixes in a single paired run:
#
#   1. VOCAB FIX (predictor._ROOT_CAUSES): added 7 performance/admission
#      root-cause tokens that were missing, which had pinned unseen-shape a1
#      near ~0.01 (object_a1 was ~0.40). Visible on the UNSEEN-shape slice.
#
#   2. FIX A (predictor prompt + adapter._summarize_investigation): make
#      opensre's investigation conclusion AUTHORITATIVE for rank-1 instead of
#      letting the predictor re-diagnose and discard it. The 2026-06-06 run
#      dropped the correct component opensre named from the predictor's top-3
#      on 15.2% of opensre+llm failures vs 5.7% for llm_alone. Visible on the
#      SEEN-shape slice as the opensre+llm − llm_alone contrast.
#
# Two arms so the contrast is directly measurable on the SAME cases:
#   opensre+llm  — full investigation, conclusion fed to predictor (Fix A)
#   llm_alone    — same predictor, empty summary (the matched control)
# (llm_alone_pure omitted to halve cost; it is not needed for either fix.)
#
# Run (needs OpenAI network egress — the agent sandbox blocks it):
#   cd opensre && set -a && source .env && set +a
#   BENCH_MIN_TOOL_CALLS=5 uv run python -m tests.benchmarks._framework.cli \
#     run tests/benchmarks/cloudopsbench/configs/cloudopsbench_fixa_validation_openai.yml --dev
#
# Cost: 40 cases x 1 LLM x 3 runs x 2 modes = 240 cells ~= $2.40 (gpt-4o).
# DEV experiment — always --dev. Exploratory, not promotable.
#
# What success looks like:
#   - UNSEEN-shape opensre+llm a1 rises from ~0.01 toward ~0.40 (vocab fix).
#   - SEEN-shape opensre+llm a1 moves ABOVE llm_alone (Fix A recovers the
#     translation loss). On the 2026-06-06 run it was 0.542 vs 0.555 (behind);
#     if Fix A works it should pull even or ahead.

benchmark: cloudopsbench

modes:
  - opensre+llm
  - llm_alone

llms:
  - gpt-4o

model_versions:
  gpt-4o: gpt-4o-2024-11-20

# 3 runs/case averages single-shot LLM noise; seed pins the SAME case set so
# the opensre+llm vs llm_alone contrast is paired.
runs_per_case: 3

workers: 1

cost_budget_usd: 25.0
seed: 42

output_dir: .bench-results/cloudopsbench_fixa_validation_openai/

pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

filters:
  # 40 seeded cases spanning BOTH shape strata so one run validates the vocab
  # fix (unseen-shape) and Fix A (seen-shape contrast). Not a publication N.
  limit: 40
  seen_shape: [true, false]
  systems: []
  fault_categories: []

report_formats:
  - json
  - markdown