tracer-cloud--opensre/tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_v1.yml

# Pre-registration — Cloud-OpsBench v1 (opensre+LLM vs paper LLM-alone baselines)
#
# COMMIT THIS BEFORE THE RUN. Integrity Mechanism 1 enforced by
# IntegrityGuard.pre_flight — the framework refuses to start a production run
# without a non-empty pre-registration on disk.
#
# Schema matches tests/benchmarks/cloudopsbench/configs/preregistrations/cloudopsbench_smoke.yml.
# YAUHEN: search for "YAUHEN" comments below — those are the spots that need
# your judgment before commit. Don't ship targets you can't defend in a PR.

cycle: cloudopsbench-v1-2026-06-05      # stamped at commit; locks the cycle identity

# --------------------------------------------------------------------------- #
# Target corpus + version pinning — all values must be confirmed at commit    #
# --------------------------------------------------------------------------- #
target_corpus:
  benchmark: cloudopsbench
  hf_dataset: tracer-cloud/cloud-ops-bench-dataset
  # Pinned HF commit SHA — matches BENCH_CORPUS_HF_REVISION in tests/benchmarks/cloudopsbench/infra/
  # (the Fargate task pulls from s3://<corpus_bucket>/<this_sha>/). Mirrored
  # via `make mirror-cloudopsbench-s3` from a developer machine.
  hf_revision: ce0ded4f196f01e176cf1d69ec15c2db42b2a677
  paper: arXiv:2603.00468v1
  case_count: 452
# opensre_sha is stamped by the runner from `git rev-parse HEAD` at run time
# and copied into provenance.json. Leave the literal placeholder here so the
# integrity gate catches a non-git checkout (e.g. dev-mode laptop run) loudly
# instead of silently emitting a report with sha = "(no-git)".
opensre_sha: REPLACE_AT_COMMIT_TIME
framework_version: 0.1.0

# --------------------------------------------------------------------------- #
# Comparison protocol — what counts as the primary contrast                     #
# --------------------------------------------------------------------------- #
# The paper (Table 4) reports each model in a "Base" (zero-shot) agent setting
# over the full 452-case corpus, single run per case, where A@k is a per-case
# MEAN and a hit requires a strict triple match of <Stage, Component, Root
# Cause> (paper § 4.2.1). Two consequences are LOCKED here:
#
#   1. PRIMARY contrast is INTERNAL and same-model: opensre+llm vs llm_alone
#      for the identical model_version. This isolates opensre's contribution
#      from the model's intrinsic ability. The paper Base/ICL numbers are an
#      EXTERNAL reference, not the primary contrast (our harness, predictor,
#      and snapshot adapter differ from the paper's CrewAI setup, so absolute
#      cross-harness numbers are not strictly comparable).
#
#   2. The honest ceiling is NOT the paper "Base" number. The paper shows a
#      cheap In-Context-Learning baseline (3 retrieved traces) lifts GPT-4o
#      0.49 -> 0.70 with NO agent framework (paper Table 5, § 4.6). Any opensre
#      lift over Base must therefore be reported ALONGSIDE the ICL number so a
#      reader can judge whether opensre's structural cost buys more than 3
#      in-context demos would. opensre must also NOT inject ICL-style
#      demonstrations into its own prompts, or the comparison is confounded.
comparison_protocol:
  primary_contrast: "opensre+llm vs llm_alone (same model_version)"
  external_reference: "paper Table 4 Base; paper Table 5 ICL (report both)"
  headline_aggregation: mean           # matches paper A@k; medians are a side view
  headline_stratum: single-shot        # consistency-selected (best-of-N) is NOT the headline
  triple_match_required: true          # <Stage/taxonomy, Component/object, Root Cause>
  ceiling_caveat: "paper ICL (GPT-4o 0.70) is the cost-equivalent baseline to beat, not Base 0.49"

# --------------------------------------------------------------------------- #
# Statistical power — committed BEFORE the run so targets are falsifiable       #
# --------------------------------------------------------------------------- #
# A target is only meaningful if the run can actually detect it. For a
# two-proportion test (α=0.05 two-sided, 80% power) against a fixed baseline
# near p≈0.5, the scenarios-per-arm required are approximately:
#     detect +0.05 absolute A@1  ->  ~1500 scenarios   (infeasible; corpus=452)
#     detect +0.10 absolute A@1  ->  ~385 scenarios    (~ full corpus)
#     detect +0.15 absolute A@1  ->  ~170 scenarios
#     detect +0.20 absolute A@1  ->  ~95 scenarios
# Repeated SEEDS do not add power — the scenario is the independent unit, so CIs
# are scenario-clustered. The N=30 pilot CANNOT resolve any of the hypothesis
# targets below; it is exploratory only. A SHIPPABLE cycle MUST run the full
# corpus (limit: null) so the +0.10..+0.15 targets become detectable.
statistical_power:
  independent_unit: scenario           # seeds are correlated; cluster on case_id
  alpha: 0.05
  power: 0.80
  min_scenarios_for_ship: 452          # full corpus; pilots (limit<452) are exploratory
  pilot_is_exploratory: true
  ci_method: scenario-clustered bootstrap (2000 resamples)

# --------------------------------------------------------------------------- #
# Provenance requirements — what makes a run promotable to a baseline           #
# --------------------------------------------------------------------------- #
# The audit found a run with opensre_sha="(no-git)" and dev-mode gates skipped.
# Such runs are valid for iteration but MUST NOT be promoted to a baseline.
provenance_requirements:
  committed_checkout_required: true    # opensre_sha must be a real SHA, not "(no-git)"/"-dirty"
  dev_mode_forbidden_for_baseline: true  # integrity gates must run (no --dev) to ship
  prereg_sha_pinned_in_report: true    # provenance.json must carry this file's sha256
  model_version_pinned: true           # gpt-4o / gpt-5 resolved to dated snapshots, verified post-run

# --------------------------------------------------------------------------- #
# Per-model hypotheses — what we expect, why, before seeing any data           #
# --------------------------------------------------------------------------- #
# Each hypothesis names ONE failure mode the paper identified + the opensre
# mechanism that should address it + the expected lift. Lift values are A@1
# unless noted. confidence is 0.0–1.0; cost_to_implement is engineer-days
# (0 because all four mechanisms are already shipped — these are predictions,
# not work items).
hypotheses:
  - id: H001
    model: claude-4-sonnet
    paper_a1: 0.50
    paper_failure_mode: |
      Paper Table 4: Claude-4-Sonnet ZTDR=0.32, meaning 32% of episodes
      generated a final result without performing valid tool invocations.
      "Speculative shortcuts" — claims a root cause from generic alert
      keywords rather than evidence (paper § 4.5).
    mechanism: |
      opensre's evidence-gathering pipeline requires verified tool output
      at each stage gate. Speculative leaps with no tool calls are blocked
      at the planner/verifier transition. ZTDR should fall to near 0; A@1
      lifts as evidence-grounded diagnoses replace speculation.
    target_a1: 0.65                    # YAUHEN: confirm — paper 0.50 → target 0.65 (+0.15 absolute)
    expected_lift:
      a1: 0.15                         # YAUHEN: edit if you want a different target
      ztdr_reduction: 0.30             # absolute drop in ZTDR (0.32 → ~0.02)
    expected_regression:
      steps: 1.5                       # opensre's stage gates add steps; honest cost
    confidence: 0.70                   # YAUHEN: adjust — high because mechanism is direct
    cost_to_implement: 0
    notes: |
      Highest-confidence lift. Failure mode (speculation) maps 1:1 to opensre
      structural strength (evidence-gating). Risk: if the bench-corpus alert
      keywords are too specific, Claude may not need to speculate.

  - id: H002
    model: gpt-4o
    paper_a1: 0.49
    paper_failure_mode: |
      Paper Table 4: GPT-4o averages 5.67 steps with MTTI=23.27s — shortest
      trajectories among large models but lowest A@1. "Premature convergence":
      stops investigating too early (paper § 4.3).
    mechanism: |
      opensre's stage-gated investigation requires meeting evidence
      thresholds before transitioning to final diagnosis. Early termination
      is blocked by the planner. Steps should rise; A@1 follows.
    target_a1: 0.65                    # YAUHEN: confirm — paper 0.49 → target 0.65 (+0.16)
    expected_lift:
      a1: 0.16                         # YAUHEN: edit
      steps_increase: 3.0              # 5.67 → ~8.5 (closer to DeepSeek's 10)
    expected_regression:
      mtti: 15.0                       # latency cost of more thorough investigation
    confidence: 0.65                   # YAUHEN: adjust
    cost_to_implement: 0
    notes: |
      Premature convergence is exactly what stage gates prevent. Risk: if
      the underlying model literally lacks the reasoning depth to fill the
      extra steps, lift will be smaller than predicted.

  - id: H003
    model: gpt-5
    paper_a1: 0.67
    paper_failure_mode: |
      Paper Table 4: GPT-5 has Relevance=0.65 (highest among large models) but
      Coverage=0.77 (below DeepSeek's 0.88). "Relevance-Centric Strategy" —
      precise tool selection but narrow exploration (paper § 4.4).
    mechanism: |
      opensre's planner prompts coverage-style exploration before commitment.
      Coverage should rise toward DeepSeek's 0.88; A@1 follows as more
      ground-truth tools get exercised.
    target_a1: 0.78                    # YAUHEN: confirm — paper 0.67 → target 0.78 (+0.11)
    expected_lift:
      a1: 0.11                         # YAUHEN: edit
      coverage_increase: 0.10
    expected_regression:
      steps: 2.0                       # exploration costs steps
    confidence: 0.50                   # YAUHEN: adjust — lower; GPT-5 is already strong
    cost_to_implement: 0
    notes: |
      Lift is constrained by ceiling — GPT-5 is already the second-strongest
      paper model. Mechanism is real but headroom is smaller than for
      GPT-4o or Claude.

  - id: H004
    model: deepseek-v3.2
    paper_a1: 0.73
    paper_failure_mode: |
      No clear failure mode — DeepSeek is the paper SOTA (A@1=0.73, 10 steps,
      Coverage=0.88, RAR=0.11 functional self-correction). Already does what
      opensre is designed to enforce structurally.
    mechanism: |
      opensre may not add much on top of a model that already explores
      thoroughly and self-corrects. Expected lift comes from edge cases where
      DeepSeek's open-loop exploration misses what opensre's stage gates
      catch — primarily the Performance category (paper A@1<0.36).
    target_a1: 0.78                    # YAUHEN: confirm — paper 0.73 → target 0.78 (+0.05, modest)
    expected_lift:
      a1: 0.05                         # YAUHEN: edit if you want more or less
    expected_regression: {}            # paper already pays the steps/coverage cost
    confidence: 0.40                   # YAUHEN: adjust — lowest confidence
    cost_to_implement: 0
    notes: |
      Modest target. Near-ceiling for the corpus. Negative result here
      (lift ≤ 0.02) would be the cleanest evidence that opensre doesn't
      benefit already-strong agents — a publishable finding either way.

# --------------------------------------------------------------------------- #
# Adversarial pre-mortem (Mechanism 2) — where we expect to LOSE                #
# --------------------------------------------------------------------------- #
# Committed BEFORE results. If any of these turn out true, the report's
# negative_results section must say so explicitly.
expected_failures:
  - "DeepSeek-V3.2 lift will be ≤ 0.05 — paper-leader near ceiling"
  - "On the Performance category (paper A@1 < 0.36 universal), no model will reach A@1 ≥ 0.50"
  - "On the Admission category (paper A@1 < 0.36 universal), opensre's stage gates may slow down small-context models without helping accuracy — runs_per_case variance will be visible"
  - "Per-cell cost_usd will be 0 (per-cell delta capture is documented TODO); aggregate cost in report.json.cost is authoritative"
  - "Validity metrics (citation_grounding, entity_existence, kubectl_actionability) may drop slightly for models that wander more — that's a real cost of broader exploration, not a regression to hide"

# --------------------------------------------------------------------------- #
# Anti-overfit gate (BDIL Phase F) — committed BEFORE any optimization         #
# --------------------------------------------------------------------------- #
# 80% of cases optimize-against, 20% held-out eval-only. Held-out cases are
# selected by seeded random sample of case_id from the full corpus.
generalization_gate:
  optimize_split_frac: 0.80
  held_out_split_frac: 0.20
  held_out_seed: 42                    # SEPARATE from config.seed; never share
  ship_threshold: 0.70                 # held_out_lift / optimize_lift >= 0.70 → ship
  rollback_threshold: 0.30             # held_out_lift / optimize_lift < 0.30 → rollback
  abort_threshold:
    held_out_a1_drop: 0.05             # if held-out A@1 drops 5pp vs paper, abort the cycle
  notes: |
    The held-out split is set ONCE at commit time and never re-rolled.
    Inspecting held-out scores during optimization is treated as a
    contamination event and invalidates the cycle.

# --------------------------------------------------------------------------- #
# Stopping rules                                                                #
# --------------------------------------------------------------------------- #
stopping_rules:
  success: |
    Cycle ships if: (a) ≥3 of 4 model hypotheses meet their A@1 target
    within absolute 0.03, AND (b) generalization_gate.ship_threshold holds,
    AND (c) no expected_failure has been silently ignored in the report.
  abort: |
    Cycle aborts if: (a) cost_budget_usd exceeded, OR (b) opensre_sha drift
    detected mid-run, OR (c) model_versions mismatch caught by pre-flight,
    OR (d) >20% of cells fail with `ok=False`.
  partial: |
    On clean abort (CostBudgetExceeded), the partial-completion report is
    published with the same integrity gates. Numbers from a partial run are
    NEVER promoted to a baseline; only complete runs are baselines.

# --------------------------------------------------------------------------- #
# Contamination check declaration (Mechanism 7)                                 #
# --------------------------------------------------------------------------- #
contamination_check:
  data_contamination_checked: true     # adapter flag is set
  declared_by: yauhen
  date: 2026-06-05
  evidence: |
    Cloud-OpsBench was published 2026-02-28 (arXiv:2603.00468v1). All four
    models in this run have training cutoffs documented as PRIOR to this
    date — therefore none could have seen the corpus during pre-training.
    No explicit anti-contamination check was run on per-case generation;
    contamination risk is bounded by the publication-vs-cutoff date gap.
  caveats: |
    If any model in the grid is later updated to a snapshot with a training
    cutoff AFTER 2026-02-28, this declaration becomes invalid and the cycle
    must be re-pre-registered.

# --------------------------------------------------------------------------- #
# Conflict-of-interest disclosure (Mechanism 10) — carried into every report   #
# --------------------------------------------------------------------------- #
coi_disclosure: |
  This benchmark run was authored, executed, and interpreted by Yauhen
  Bichel, who is also the primary author of opensre. This is a structural
  conflict of interest mitigated by: (a) this pre-registration committed
  before the run, (b) per-stratum reporting (seen-shape vs unseen-shape),
  (c) required negative-results section, (d) external replication of at
  least one cell before any public claim, (e) standardization-by-pinning
  of every parameter that affects results (model_version, opensre_sha,
  HF dataset revision, seed). Reviewers are encouraged to reproduce any
  cell independently using the artifacts published in raw_artifacts_dir.

# --------------------------------------------------------------------------- #
# What we will NOT do post-results (anti-p-hacking discipline)                 #
# --------------------------------------------------------------------------- #
will_not_do_post_results:
  - "Re-weight metrics to favor the family where lift is largest"
  - "Drop or re-bucket cases that hurt the headline number"
  - "Re-run failed cells selectively (re-runs are all-or-nothing per cell × model × run_index)"
  - "Optimize prompts against held-out cases between pre-registration and ship decision"
  - "Promote a partial-completion run to a baseline"