tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/cloudopsbench_floor0_ablation_openai.yml

# Cloud-OpsBench — floor=0 ablation (cheap, gpt-4o, 2 arms).
#
# Experiment ID: ``exp_floor0_ablation``  (pair against ``exp_definitive`` which
# is the in-flight 452-case run at floor=5).
#
# Hypothesis: ``MIN_TOOL_CALLS=5`` forces extra tool calls past correct
# conclusions. Extra calls bring noise (irrelevant logs, misleading
# connectivity probes, unrelated services in tool outputs). That noise
# pollutes the investigation summary that the predictor LLM has to thread
# through. Concrete example: boutique/startup/36 (incorrect_image_reference
# on cartservice) — the agent correctly identifies the bad image at call 2-3,
# but the floor forces it to keep going. Subsequent
# CheckServiceConnectivity returns "registry unreachable" (because the pod
# hasn't started), pushing the predictor toward image_registry_dns_failure.
#
# What this run does: same 40 cases, same arms, same predictor, same scoring
# as ``cloudopsbench_fixa_validation_openai.yml``. ONLY change: set
# BENCH_MIN_TOOL_CALLS=0 at launch so the LLM stops when it decides instead
# of being forced past 5 calls.
#
# Pre-registered predictions:
#   - Most likely (60%): a1 ≈ floor=5 baseline ± 0.02 — opensre's structural
#     mechanisms simply don't differentiate from pure-LLM on this case mix,
#     floor or no floor.
#   - Possible (25%): floor=0 a1 > floor=5 a1 by 0.02-0.05 — confirms the
#     "forced exploration adds noise" hypothesis. Adaptive floor would be
#     the next iteration.
#   - Less likely (15%): floor=0 a1 < floor=5 a1 by 0.02-0.05 — investigation
#     depth is buying real signal that the floor protects.
#
# Decision rule (committed BEFORE seeing the result):
#   - If floor=0 wins by ≥0.03 with CI excluding 0 → ship adaptive-floor
#     experiment as the next bench (estimated next experiment cost: ~$15).
#   - If CI includes 0 → confirms equivalence; floor is not the lever.
#     Pivot bench work to corpus design.
#   - If floor=0 loses by ≥0.03 → confirms MIN_TOOL_CALLS=5 calibration;
#     no further floor sweeps.
#
# Cost: 40 cases × 1 LLM × 3 runs × 2 modes = 240 cells ~= $4 (gpt-4o).
# Wall time: ~1h at workers=1 (matches the prior fixa baseline).
#
# Run (must set BENCH_MIN_TOOL_CALLS=0 explicitly; the env var override is
# the experiment):
#   cd opensre && set -a && source .env && set +a
#   BENCH_MIN_TOOL_CALLS=0 uv run python -m tests.benchmarks._framework.cli \
#     run tests/benchmarks/cloudopsbench/configs/cloudopsbench_floor0_ablation_openai.yml --dev
#
# For Fargate: set BENCH_MIN_TOOL_CALLS=0 in the ECS task definition
# container env vars before dispatching. Image :f063cb1 already supports
# the env var.

benchmark: cloudopsbench

modes:
  - opensre+llm
  - llm_alone_pure

llms:
  - gpt-4o

model_versions:
  gpt-4o: gpt-4o-2024-11-20

runs_per_case: 3

# workers=1 matches the prior fixa baseline at floor=5 (12:09Z run).
# Apples-to-apples comparison requires same concurrency profile.
workers: 1

cost_budget_usd: 25.0
seed: 42

output_dir: .bench-results/cloudopsbench_floor0_ablation_openai/

pre_registration_path: tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

filters:
  # 40 seeded cases — IDENTICAL to cloudopsbench_fixa_validation_openai and
  # the 12:09Z baseline so the floor effect is the only variable that changed.
  limit: 40
  seen_shape: [true, false]
  systems: []
  fault_categories: []

report_formats:
  - json
  - markdown