tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_structured_outputs_smoke_openai.yml

# Cloud-OpsBench — structured-outputs smoke (40 cases, OpenAI predictor).
#
# Experiment ID: ``exp_structured_outputs`` (smoke phase)
#
# Tests the OpenAI grammar-constrained predictor variant on the 40-case
# trimmed-prompt slice. Single-variable change vs ``cloudopsbench_trimmed_prompt_openai.yml``:
# ``predictor_variant: structured`` (this file) vs ``default`` (baseline).
#
# Mechanism: OpenAI's ``response_format`` enforces the closed-vocabulary
# ``root_cause`` and ``fault_taxonomy`` enums at the API layer. The LLM
# literally cannot emit out-of-enum tokens, so adjacent-vocabulary failure
# cells (e.g. ``mysql_invalid_port`` → ``db_connection_exhaustion``) are
# blocked at the sampler, not at the parser.
#
# Run:
#   cd opensre && set -a && source .env && set +a
#   uv run python -m tests.benchmarks._framework.cli \
#     run tests/benchmarks/cloudopsbench/configs/cloudopsbench_structured_outputs_smoke_openai.yml --dev
#
# Cost: ~$4 (mirrors the prior 40-case smoke; structured outputs adds no
# extra token cost — it constrains sampling, not prompt length).
# Wall time: ~30-45 min Fargate at workers=1.
#
# Pre-registered decision rule for advancing to full-N:
#   OBJECT_HIT_RC_MISS share of opensre+llm losses (n=40 smoke) drops
#   ≥3pp vs the prior baseline (24% → ≤21%). The smoke is too small to
#   resolve A@1 lift confidently; the share metric is the direct
#   mechanism check.

benchmark: cloudopsbench

modes:
  - opensre+llm
  - llm_alone
  - llm_alone_pure

llms:
  - gpt-4o

model_versions:
  gpt-4o: gpt-4o-2024-11-20

runs_per_case: 3
workers: 1
cost_budget_usd: 25.0
seed: 42

output_dir: .bench-results/cloudopsbench_structured_outputs_smoke_openai/

pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/exp_structured_outputs_v1.yml

filters:
  # 40 seeded cases — identical slice to the trimmed-prompt smoke so the
  # predictor variant is the only variable.
  limit: 40
  seen_shape: [true, false]
  systems: []
  fault_categories: []

# Floor=0 + trimmed prompt — matches the current bench default established
# by the post-vocab-fix run. The predictor variant is the only change here.
min_tool_calls: 0
agent_variant: trimmed_prompt

# The lever this experiment is testing. Swaps the predictor's API call to
# OpenAI structured outputs (response_format + Pydantic schema) instead of
# the default text-emit. OpenAI-only; cross-field lint enforces.
predictor_variant: structured

report_formats:
  - json
  - markdown