tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_control_openai.yml

# Cloud-OpsBench — control-arm contrast at the chosen floor (cheap, gpt-4o).
#
# Purpose: the floor sweep settled MIN_TOOL_CALLS=5 as the default. The one
# number still missing is ATTRIBUTION — how much of opensre+llm's a1 is the
# agent's policy vs. the model's intrinsic skill. This config runs the SAME
# 30-case paired pilot set (seed 42, seen-shape) across all three arms so the
# contrast is apples-to-apples:
#
#   (opensre+llm)    - (llm_alone)      = lift from the MIN_TOOL_CALLS floor
#   (opensre+llm)    - (llm_alone_pure) = lift from opensre's full stack
#   (llm_alone)      - (llm_alone_pure) = lift from opensre's prompt alone
#
# Run at the committed default floor (stamped into provenance):
#
#   BENCH_MIN_TOOL_CALLS=5 uv run python -m tests.benchmarks._framework.cli \
#       run tests/benchmarks/cloudopsbench/configs/cloudopsbench_control_openai.yml --dev
#
# The floor only affects opensre+llm; llm_alone / llm_alone_pure have no floor,
# so the baselines are identical regardless of BENCH_MIN_TOOL_CALLS.
#
# Cost: 270 cells (30 cases x 1 LLM x 3 runs x 3 modes) at ~$0.01/cell ≈ $3-5.
# DEV experiment (small, single-shape) — pass --dev so the power/generalization
# gates don't block it. The powered full-corpus contrast remains
# cloudopsbench_v1_openai.yml.

benchmark: cloudopsbench

modes:
  - opensre+llm
  - llm_alone
  - llm_alone_pure

llms:
  - gpt-4o

model_versions:
  gpt-4o: gpt-4o-2024-11-20

runs_per_case: 3

workers: 1

cost_budget_usd: 25.0
seed: 42

output_dir: .bench-results/cloudopsbench_control_openai/

pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

filters:
  # Same 30 seeded cases as the floor sweep (seed pins the selection) so the
  # control contrast is paired with the sweep results.
  limit: 30
  seen_shape: [true]
  systems: []
  fault_categories: []

report_formats:
  - json
  - markdown