tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_postpatch_anthropic.yml

# Cloud-OpsBench — full N, post-patch Anthropic run (publication-grade).
#
# Promotes the 2026-06-07 vocab-pilot signal to the full corpus on Claude.
# The pilot (cloudopsbench_vocabpilot_anthropic.yml, 60 cells) measured
# unseen-shape a1 = 0.467 vs the 2026-06-06 OpenAI baseline of 0.014 — a 33×
# lift, comfortably above the 0.30 green-light threshold. This config
# scales that signal to the full 452 cases × 3 seeds on Claude to produce
# a publication-grade absolute number.
#
# What's under test:
#   - The 7 vocab additions in predictor._ROOT_CAUSES (pod_network_delay,
#     pod_cpu_overload, namespace_*_quota_exceeded)
#   - The scope rule in predictor._build_system_prompt (namespace_* root_cause
#     → namespace/<X> object, multi-service same-namespace → namespace-scope)
#
# What's NOT under test in this run:
#   - The opensre+llm vs llm_alone contrast. Both arms share predictor.py
#     so the fixes lift both equally — that contrast was already shown to be
#     null on 06-06. Adding the two baseline arms ~3× the spend without
#     changing the headline. Run them in a follow-up if needed for publication.
#   - The gpt-4o paper parity. OpenAI credit was exhausted as of 06-07;
#     Claude here gives us the absolute number on a paper-baseline-comparable
#     model (Claude-4-Sonnet's paper baseline is 0.50, near gpt-4o's 0.49).
#
# Run (NO --dev — this is publication-grade; IntegrityGuard required):
#   set -a && source .env && set +a && \
#   uv run python -m tests.benchmarks._framework.cli \
#     run tests/benchmarks/cloudopsbench/configs/cloudopsbench_postpatch_anthropic.yml
#
# Cost projection: pilot was 60 cells in 52 min on workers=2. At full N:
#   - 452 cases × 3 seeds × 1 mode × 1 LLM = 1,356 cells
#   - Per-cell wall time ~50s; at workers=4 → ~5-6 h wall time
#   - Token spend ~$50-80 at claude-sonnet-4-5 rates (rough; Anthropic
#     telemetry wasn't surfaced in the pilot report)
#   - $200 budget gives ~3× headroom for unexpected long-tail cases
#
# Pilot baseline to compare against (when reading the report):
#   - unseen-shape a1 = 0.467  (60 cells, opensre+llm/claude-4-sonnet, --dev)
#   - unseen-shape object_a1 = 0.667
#   - boutique stratum a1 = 0.718  (where scope rule dominates)
#   - trainticket stratum a1 = 0.000  (vocab fix unlocks family, not specific token)

benchmark: cloudopsbench

modes:
  - opensre+llm

llms:
  - claude-4-sonnet

model_versions:
  claude-4-sonnet: claude-sonnet-4-5-20250929

runs_per_case: 3

# workers=4 is safe for Anthropic tier-1 (generous TPM); pilot at workers=2
# ran cleanly with 12 trim events across 60 cells (predictable, not 429-driven).
# Doubling concurrency cuts wall time roughly in half.
workers: 4

cost_budget_usd: 200.0
seed: 42

output_dir: .bench-results/cloudopsbench_postpatch_anthropic/

# Same pre-registration as the v1 openai grid — both runs are testing the
# same hypotheses on the same corpus, just on different model providers.
pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

filters:
  # Full corpus, both shape strata (the headline number the pilot is scaling).
  seen_shape: [true, false]
  systems: []
  fault_categories: []

report_formats:
  - json
  - markdown
  - html