tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_structured_outputs_openai.yml

# Cloud-OpsBench — structured-outputs full-N (n=353, three-arm, OpenAI predictor).
#
# Experiment ID: ``exp_structured_outputs`` (definitive phase)
#
# Tests the OpenAI grammar-constrained predictor variant at full N. Single-
# variable change vs ``cloudopsbench_definitive_trimmed_prompt_openai.yml``:
# ``predictor_variant: structured`` (this file) vs ``default`` (baseline).
#
# Pre-registered targets (full detail in
# preregistrations/exp_structured_outputs_v1.yml):
#   - aggregate A@1 lift ≥ +0.03 with 95% CI excluding 0
#   - Runtime A@1 lift ≥ +0.05 (point estimate; n=141 CI may overlap 0)
#   - paired Δ(opensre+llm − llm_alone) ≥ +0.07 floor preserved
#   - OBJECT_HIT_RC_MISS share of losses ≤ 15% (direct mechanism check)
#   - no regression on any per-stratum A@1 floor
#   - held-out generalization gate: held_out_lift / optimize_lift ≥ 0.70
#
# Anti-overfit guards (full detail in pre-reg):
#   - Schema-vocabulary invariant tests (already in test_predictor_structured.py)
#   - Per-system lift uniformity (boutique vs trainticket within ±0.05)
#   - Per-stratum lift uniformity (no category > 2× median lift)
#   - Per-case attribution: flipped-to-win cluster concentration ≤ 0.60
#   - A/A consistency: same config two seeds, aggregate diff < 0.02
#
# Run:
#   cd opensre && set -a && source .env && set +a
#   uv run python -m tests.benchmarks._framework.cli \
#     run tests/benchmarks/cloudopsbench/configs/cloudopsbench_structured_outputs_openai.yml --dev
#
# Cost: ~$50 (mirrors the prior three-arm full-N at the same model + N).
# Wall time: ~3.5h Fargate at workers=4.

benchmark: cloudopsbench

modes:
  - opensre+llm
  - llm_alone
  - llm_alone_pure

llms:
  - gpt-4o

model_versions:
  gpt-4o: gpt-4o-2024-11-20

runs_per_case: 3
workers: 4
cost_budget_usd: 100.0
seed: 42

output_dir: .bench-results/cloudopsbench_structured_outputs_openai/

pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/exp_structured_outputs_v1.yml

filters:
  # Full corpus, both shape strata — identical to the prior
  # exp_definitive_trimmed_prompt baseline.
  seen_shape: [true, false]
  systems: []
  fault_categories: []

# Floor=0 + trimmed prompt — matches the current bench default.
min_tool_calls: 0
agent_variant: trimmed_prompt

# The lever this experiment is testing.
predictor_variant: structured

report_formats:
  - json
  - markdown
  - html