"""Render report.json + per-case artifacts into markdown + HTML. Operates on what's already on disk (``run_dir/report.json`` + ``run_dir/cases/*.json``) so it can be invoked two ways: 1. From the runner directly, right after the JSON sidecar is written 2. From the CLI as ``bench report `` — for re-rendering a finished run without re-executing anything Self-contained outputs — markdown is plain CommonMark, HTML has inline CSS only (no external dependencies, viewable in any browser). The reporting layer respects the integrity discipline: - Headline numbers ALWAYS shown with per-stratum breakdown (Mechanism 4) - Every adapter-declared metric is in the table, even when ugly (Mechanism 3) - Negative-results section verbatim from the report (Mechanism 9) - COI disclosure verbatim from the report (Mechanism 10) - Raw per-case artifact paths listed so external reviewers can verify (Mechanism 5) The reporter never aggregates away detail — that's a property of the framework, not a stylistic choice. """ from __future__ import annotations import html import json import random from collections.abc import Sequence from pathlib import Path from typing import Any # --------------------------------------------------------------------------- # # Paper reference baselines — Wang et al. 2026, "Cloud-OpsBench" Table 4 # # (arXiv:2603.00468v1), the "Base" (zero-shot) setting over the full 452-case # # corpus, single run per case. A@k there is a MEAN over cases (Eq. in §4.2.1), # # not a median, and a diagnosis counts only on a strict triple match of # # . These figures are what our headline must be # # compared against — and the comparison is only valid for the single-shot, # # full-corpus stratum (see headline note). # # --------------------------------------------------------------------------- # # Full Table 4 row per model: outcome (a1/a3/tcr) + process (exact/in_order/ # any_order/rel/cov) + efficiency/robustness (steps/iac/rar/ztdr). MTTI is # deliberately omitted — wall-clock seconds, hardware/provider dependent, and # not measured in this harness (see _NON_COMPARABLE_METRICS). _PAPER_BASELINE: dict[str, dict[str, float]] = { "gpt-4o": { "a1": 0.49, "a3": 0.55, "tcr": 0.99, "exact": 0.14, "in_order": 0.45, "any_order": 0.46, "rel": 0.63, "cov": 0.78, "steps": 5.67, "iac": 0.27, "rar": 0.02, "ztdr": 0.02, }, # noqa: E501 "gpt-5": { "a1": 0.67, "a3": 0.75, "tcr": 0.99, "exact": 0.16, "in_order": 0.38, "any_order": 0.48, "rel": 0.65, "cov": 0.77, "steps": 5.57, "iac": 0.04, "rar": 0.05, "ztdr": 0.04, }, # noqa: E501 "claude-4-sonnet": { "a1": 0.50, "a3": 0.54, "tcr": 0.98, "exact": 0.05, "in_order": 0.24, "any_order": 0.25, "rel": 0.46, "cov": 0.52, "steps": 4.25, "iac": 0.12, "rar": 0.05, "ztdr": 0.32, }, # noqa: E501 "deepseek-v3.2": { "a1": 0.73, "a3": 0.79, "tcr": 0.99, "exact": 0.0, "in_order": 0.53, "any_order": 0.63, "rel": 0.43, "cov": 0.88, "steps": 10.0, "iac": 0.25, "rar": 0.11, "ztdr": 0.0, }, # noqa: E501 "qwen3-235b": { "a1": 0.50, "a3": 0.53, "tcr": 0.96, "exact": 0.13, "in_order": 0.38, "any_order": 0.41, "rel": 0.55, "cov": 0.67, "steps": 5.34, "iac": 0.22, "rar": 0.06, "ztdr": 0.17, }, # noqa: E501 "qwen3-14b": { "a1": 0.34, "a3": 0.43, "tcr": 0.82, "exact": 0.04, "in_order": 0.31, "any_order": 0.42, "rel": 0.63, "cov": 0.71, "steps": 5.82, "iac": 0.40, "rar": 0.10, "ztdr": 0.0, }, # noqa: E501 "qwen3-8b": { "a1": 0.21, "a3": 0.23, "tcr": 0.92, "exact": 0.01, "in_order": 0.15, "any_order": 0.20, "rel": 0.36, "cov": 0.47, "steps": 5.46, "iac": 0.40, "rar": 0.16, "ztdr": 0.27, }, # noqa: E501 } # Paper Table 5 — In-Context Learning (3 retrieved diagnostic traces, NO agent # framework). The cost-equivalent baseline opensre actually has to beat: a few # in-context demos lift GPT-4o 0.49 -> 0.70 with no orchestration. Only the # three models the paper ran under ICL are present. _PAPER_ICL: dict[str, dict[str, float]] = { "gpt-4o": { "a1": 0.70, "a3": 0.75, "tcr": 0.97, "exact": 0.28, "in_order": 0.49, "any_order": 0.52, "rel": 0.67, "cov": 0.76, "steps": 4.40, "iac": 0.08, "rar": 0.0, "ztdr": 0.13, }, # noqa: E501 "qwen3-235b": { "a1": 0.59, "a3": 0.63, "tcr": 0.98, "exact": 0.27, "in_order": 0.52, "any_order": 0.54, "rel": 0.57, "cov": 0.66, "steps": 3.11, "iac": 0.09, "rar": 0.03, "ztdr": 0.30, }, # noqa: E501 "qwen3-14b": { "a1": 0.71, "a3": 0.75, "tcr": 0.99, "exact": 0.11, "in_order": 0.44, "any_order": 0.59, "rel": 0.70, "cov": 0.86, "steps": 6.29, "iac": 0.29, "rar": 0.11, "ztdr": 0.0, }, # noqa: E501 } # Metrics defined identically in the paper (Table 4) — the only set for which a # head-to-head number against the published baseline is meaningful. MTTI is # excluded on purpose (see _NON_COMPARABLE_METRICS). _PAPER_COMPARABLE_METRICS = [ "a1", "a3", "exact", "in_order", "any_order", "rel", "cov", "steps", "iac", "rar", "ztdr", ] # Computed by our scorer but NOT comparable to the paper, with the reason. # Surfaced as a footnote so a reader doesn't mistake a structural 0 (or a # saturated 1.0) for a result. _NON_COMPARABLE_METRICS = { "mtti": "measured wall-clock seconds to diagnosis, but hardware/provider/" "network dependent — useful for internal A/B (e.g. floor sweeps), not a " "like-for-like number against the paper's setup", "tcr": "saturated at 1.0 — the predictor always emits structured output, so this " "does not track the paper's crash/schema-violation rate", } # opensre-only instrumentation. Useful as internal diagnostics, but NOT present # in the paper, so they are reported in a separate panel to avoid implying a # comparison that doesn't exist. # L0 investigation-native metrics (opensre prose, keyword parser). Distinct from # L1 ``a1`` which scores the predictor's rank-1 formalization. _L0_INVESTIGATION_METRICS = [ "investigation_a1", "investigation_partial_a1", "investigation_object_a1", "translation_loss", ] _L0_CI_METRICS = frozenset({"investigation_a1", "investigation_object_a1"}) _OPENSRE_ONLY_METRICS = [ "partial_a1", "partial_a3", "object_a1", "object_a3", "citation_grounding_rate", "entity_existence_rate", "kubectl_actionability_rate", ] def _match_paper_row(table: dict[str, dict[str, float]], llm: str) -> dict[str, float] | None: """Best-effort match of a run's LLM label to a row in a paper table.""" key = llm.strip().lower() if key in table: return table[key] for name, row in table.items(): if name in key or key in name: return row return None def _match_paper_baseline(llm: str) -> dict[str, float] | None: """Paper Table 4 Base (zero-shot) row for this LLM, if any.""" return _match_paper_row(_PAPER_BASELINE, llm) def _match_paper_icl(llm: str) -> dict[str, float] | None: """Paper Table 5 ICL row for this LLM, if any (only 3 models exist).""" return _match_paper_row(_PAPER_ICL, llm) # --------------------------------------------------------------------------- # # Public API # # --------------------------------------------------------------------------- # def render_report_dir( run_dir: Path, formats: Sequence[str] | None = None, ) -> dict[str, Path]: """Render artifacts under ``run_dir`` to the requested formats. Args: run_dir: directory containing ``report.json`` and ``cases/``. formats: subset of {"markdown", "html"}; defaults to both. Returns: Mapping format -> path of the rendered artifact. Raises: FileNotFoundError: if ``report.json`` is missing. """ formats = formats or ["markdown", "html"] report_path = run_dir / "report.json" if not report_path.exists(): raise FileNotFoundError(f"Missing {report_path}; run hasn't produced a report yet") report = json.loads(report_path.read_text(encoding="utf-8")) cases_dir = run_dir / "cases" cells = _load_cells(cases_dir) if cases_dir.exists() else [] provenance = _load_provenance(run_dir / "provenance.json") out: dict[str, Path] = {} if "markdown" in formats: md_path = run_dir / "report.md" md_path.write_text(_render_markdown(report, cells, provenance), encoding="utf-8") out["markdown"] = md_path if "html" in formats: html_path = run_dir / "report.html" html_path.write_text(_render_html(report, cells, provenance), encoding="utf-8") out["html"] = html_path return out def _load_provenance(path: Path) -> dict[str, Any] | None: """Optional — provenance is recommended but not required for re-rendering.""" if not path.exists(): return None try: loaded = json.loads(path.read_text(encoding="utf-8")) except json.JSONDecodeError: return None if isinstance(loaded, dict): return loaded return None # --------------------------------------------------------------------------- # # Loading # # --------------------------------------------------------------------------- # def _load_cells(cases_dir: Path) -> list[dict[str, Any]]: """Load every per-case artifact in ``cases_dir`` as a dict.""" cells: list[dict[str, Any]] = [] for path in sorted(cases_dir.glob("*.json")): try: cells.append(json.loads(path.read_text(encoding="utf-8"))) except json.JSONDecodeError: # Skip corrupt artifacts but record path so the report shows the gap cells.append({"_load_error": str(path)}) return cells # --------------------------------------------------------------------------- # # Aggregation helpers # # --------------------------------------------------------------------------- # def _per_cell_metric(cells: list[dict[str, Any]], metric: str) -> list[float]: """Pull one metric across all cells as a flat float list.""" out: list[float] = [] for cell in cells: value = cell.get("score", {}).get("metrics", {}).get(metric) if isinstance(value, (int, float)): out.append(float(value)) return out def _cell_mode(cell: dict[str, Any]) -> str: return cell.get("run", {}).get("mode", "(unknown)") def _cell_category(cell: dict[str, Any]) -> str: return cell.get("case", {}).get("metadata", {}).get("fault_category", "(unknown)") def _cells_by_llm_mode( cells: list[dict[str, Any]], ) -> dict[str, dict[str, list[dict[str, Any]]]]: """Group cells as ``{llm: {mode: [cells]}}``. Splitting on mode matters once the ``llm_alone`` control arm runs: pooling both modes into one LLM bucket would silently average the opensre+llm result with its own baseline. """ out: dict[str, dict[str, list[dict[str, Any]]]] = {} for cell in cells: if "_load_error" in cell: continue llm = cell.get("run", {}).get("llm", "(unknown)") mode = _cell_mode(cell) out.setdefault(llm, {}).setdefault(mode, []).append(cell) return out def _paired_scenario_deltas( cells: list[dict[str, Any]], llm: str, metric: str, mode_a: str, mode_b: str, ) -> list[float]: """Per-scenario ``metric(mode_a) − metric(mode_b)`` for one LLM. Only scenarios present in BOTH modes contribute (a paired difference), so the control delta isolates opensre's policy from scenario mix. Seeds within a scenario are averaged before differencing. """ a: dict[str, list[float]] = {} b: dict[str, list[float]] = {} for cell in cells: if cell.get("run", {}).get("llm") != llm: continue value = cell.get("score", {}).get("metrics", {}).get(metric) if not isinstance(value, (int, float)): continue case_id = cell.get("case", {}).get("case_id", "(unknown)") mode = _cell_mode(cell) if mode == mode_a: a.setdefault(case_id, []).append(float(value)) elif mode == mode_b: b.setdefault(case_id, []).append(float(value)) deltas: list[float] = [] for case_id in a.keys() & b.keys(): mean_a = sum(a[case_id]) / len(a[case_id]) mean_b = sum(b[case_id]) / len(b[case_id]) deltas.append(mean_a - mean_b) return deltas def _scenario_means(cells: list[dict[str, Any]], metric: str) -> list[float]: """Collapse per-seed cells to one value per scenario (case_id). The benchmark runs multiple seeds per scenario; those repeats are *correlated*, not independent samples. Treating each run as an independent observation under-states the variance and inflates significance. The scenario is the independent unit, so we average the seeds within each scenario first and return one value per scenario. """ buckets: dict[str, list[float]] = {} for cell in cells: value = cell.get("score", {}).get("metrics", {}).get(metric) if not isinstance(value, (int, float)): continue case_id = cell.get("case", {}).get("case_id", "(unknown)") buckets.setdefault(case_id, []).append(float(value)) return [sum(vs) / len(vs) for vs in buckets.values() if vs] def _mean_with_ci( scenario_values: list[float], *, iters: int = 2000, seed: int = 12345, ) -> tuple[float, float, float, int]: """Mean + 95% scenario-clustered bootstrap CI. Resamples scenarios (not runs) with replacement so the interval reflects between-scenario variability — the level at which the paper's A@k is a per-case mean. Returns ``(mean, ci_low, ci_high, n_scenarios)``. With fewer than 2 scenarios a CI is undefined, so low==high==mean. """ n = len(scenario_values) if n == 0: return 0.0, 0.0, 0.0, 0 mean = sum(scenario_values) / n if n < 2: return mean, mean, mean, n rng = random.Random(seed) boot_means: list[float] = [] for _ in range(iters): sample_sum = 0.0 for _ in range(n): sample_sum += scenario_values[rng.randrange(n)] boot_means.append(sample_sum / n) boot_means.sort() lo = boot_means[int(0.025 * iters)] hi = boot_means[min(iters - 1, int(0.975 * iters))] return mean, lo, hi, n # --------------------------------------------------------------------------- # # Decomposition — "where does the accuracy go?" (shared md/html data) # # --------------------------------------------------------------------------- # _PRIMARY_MODE = "opensre+llm" _CONTROL_MODE = "llm_alone" def _control_contrast_rows( cells: list[dict[str, Any]], by_lm: dict[str, dict[str, list[dict[str, Any]]]], ) -> list[tuple[str, float, float, float, int, str]]: """Per-LLM paired control delta on a1: opensre+llm − llm_alone. Returns ``(llm, mean_delta, lo, hi, n_paired, verdict)`` for LLMs that ran BOTH arms. Empty when the control arm wasn't run. """ rows: list[tuple[str, float, float, float, int, str]] = [] for llm in sorted(by_lm.keys()): modes = by_lm[llm] if _PRIMARY_MODE not in modes or _CONTROL_MODE not in modes: continue deltas = _paired_scenario_deltas(cells, llm, "a1", _PRIMARY_MODE, _CONTROL_MODE) mean, lo, hi, n = _mean_with_ci(deltas) if n < 2: verdict = "too few paired scenarios" elif lo <= 0.0 <= hi: verdict = "no significant effect (CI contains 0)" elif mean > 0: verdict = "opensre helps" else: verdict = "opensre hurts" rows.append((llm, mean, lo, hi, n, verdict)) return rows def _category_a1( by_lm: dict[str, dict[str, list[dict[str, Any]]]], llm: str, mode: str, ) -> dict[str, tuple[float, int]]: """Mean a1 per fault_category for one (llm, mode), with scenario count.""" cells = by_lm.get(llm, {}).get(mode, []) by_cat: dict[str, list[dict[str, Any]]] = {} for cell in cells: by_cat.setdefault(_cell_category(cell), []).append(cell) out: dict[str, tuple[float, int]] = {} for cat, cat_cells in by_cat.items(): scen_vals = _scenario_means(cat_cells, "a1") mean, _, _, n = _mean_with_ci(scen_vals) out[cat] = (mean, n) return out def _render_decomposition_markdown( cells: list[dict[str, Any]], by_lm: dict[str, dict[str, list[dict[str, Any]]]], ) -> list[str]: """Track-2 decomposition: control delta, localization-vs-labeling, by category.""" if not by_lm: return [] lines: list[str] = [] lines.append("## Decomposition — where the accuracy goes") lines.append("") # 1. Control contrast (the number that isolates opensre's contribution) lines.append("### Control contrast — A@1(opensre+llm) − A@1(llm_alone), same model") lines.append("") contrast = _control_contrast_rows(cells, by_lm) if not contrast: lines.append( "_No control arm in this run — add `llm_alone` to `modes` so the " "delta that isolates opensre's policy (vs the model's intrinsic " "skill) can be computed. This is the single most important number._" ) else: lines.append("| LLM | Δ A@1 (paired) | 95% CI | n | verdict |") lines.append("|---|---|---|---|---|") for llm, mean, lo, hi, n, verdict in contrast: lines.append(f"| `{llm}` | {mean:+.2f} | [{lo:+.2f}, {hi:+.2f}] | {n} | {verdict} |") lines.append("") lines.append( "_Paired per-scenario difference (seeds averaged first). A CI that " "contains 0 means opensre's pipeline is statistically indistinguishable " "from bare tool-use on this model._" ) lines.append("") # 2. Localization vs labeling (opensre+llm) — is it finding the right place? has_decomp = any( _per_cell_metric(by_lm[llm].get(_PRIMARY_MODE, []), m) for llm in by_lm for m in ("object_a1", "partial_a1") ) if has_decomp: lines.append("### Localization vs labeling (opensre+llm)") lines.append("") lines.append("| LLM | a1 (triple) | object_a1 (component) | partial_a1 (relaxed) |") lines.append("|---|---|---|---|") for llm in sorted(by_lm.keys()): op_cells = by_lm[llm].get(_PRIMARY_MODE, []) vals = [] for m in ("a1", "object_a1", "partial_a1"): mean, _, _, n = _mean_with_ci(_scenario_means(op_cells, m)) vals.append(f"{mean:.2f}" if n else "—") lines.append(f"| `{llm}` | {vals[0]} | {vals[1]} | {vals[2]} |") lines.append("") lines.append( "_If `object_a1` ≫ `a1`, opensre finds the right component but " "mislabels the root cause — a predictor/translation problem, not a " "reasoning one. If both are low, the investigation missed the place._" ) lines.append("") # 3. Per fault-category A@1 (opensre+llm) — vs paper Fig. 3 difficulty categories = sorted({_cell_category(c) for c in cells if _cell_mode(c) == _PRIMARY_MODE}) if categories and any(by_lm[llm].get(_PRIMARY_MODE) for llm in by_lm): lines.append("### Per fault-category A@1 (opensre+llm)") lines.append("") header = "| LLM | " + " | ".join(categories) + " |" sep = "|" + "|".join(["---"] * (len(categories) + 1)) + "|" lines.append(header) lines.append(sep) for llm in sorted(by_lm.keys()): cat_map = _category_a1(by_lm, llm, _PRIMARY_MODE) row = [f"`{llm}`"] for cat in categories: mean, n = cat_map.get(cat, (0.0, 0)) row.append(f"{mean:.2f} (n={n})" if n else "—") lines.append("| " + " | ".join(row) + " |") lines.append("") lines.append( "_Paper Fig. 3: Startup/Runtime are easy (A@1 > 0.65), " "Admission/Performance are hard (A@1 < 0.36). Losing only where the " "paper loses = corpus difficulty; losing broadly = opensre._" ) lines.append("") return lines def _render_decomposition_html( cells: list[dict[str, Any]], by_lm: dict[str, dict[str, list[dict[str, Any]]]], esc: Any, ) -> list[str]: """HTML mirror of :func:`_render_decomposition_markdown`.""" if not by_lm: return [] parts: list[str] = [] parts.append("

Decomposition — where the accuracy goes

") # 1. Control contrast parts.append("

Control contrast — A@1(opensre+llm) − A@1(llm_alone), same model

") contrast = _control_contrast_rows(cells, by_lm) if not contrast: parts.append( '

No control arm in this run — add ' "llm_alone to modes so the delta that " "isolates opensre's policy (vs the model's intrinsic skill) can be " "computed. This is the single most important number.

" ) else: parts.append( "" "" ) for llm, mean, lo, hi, n, verdict in contrast: parts.append( f"" f'' f'' f'' ) parts.append("
LLMΔ A@1 (paired)95% CInverdict
{esc(llm)}{mean:+.2f}[{lo:+.2f}, {hi:+.2f}]{n}{esc(verdict)}
") parts.append( "

Paired per-scenario difference (seeds averaged first). " "A CI containing 0 means opensre's pipeline is statistically " "indistinguishable from bare tool-use on this model.

" ) # 2. Localization vs labeling has_decomp = any( _per_cell_metric(by_lm[llm].get(_PRIMARY_MODE, []), m) for llm in by_lm for m in ("object_a1", "partial_a1") ) if has_decomp: parts.append("

Localization vs labeling (opensre+llm)

") parts.append( "" "" "" ) for llm in sorted(by_lm.keys()): op_cells = by_lm[llm].get(_PRIMARY_MODE, []) parts.append(f"") for m in ("a1", "object_a1", "partial_a1"): mean, _, _, n = _mean_with_ci(_scenario_means(op_cells, m)) parts.append(f'' if n else "") parts.append("") parts.append("
LLMa1 (triple)object_a1 (component)partial_a1 (relaxed)
{esc(llm)}{mean:.2f}—
") parts.append( "

If object_a1 ≫ a1, opensre " "finds the right component but mislabels the root cause — a " "predictor/translation problem, not a reasoning one.

" ) # 3. Per fault-category A@1 categories = sorted({_cell_category(c) for c in cells if _cell_mode(c) == _PRIMARY_MODE}) if categories and any(by_lm[llm].get(_PRIMARY_MODE) for llm in by_lm): parts.append("

Per fault-category A@1 (opensre+llm)

") parts.append("") for cat in categories: parts.append(f"") parts.append("") for llm in sorted(by_lm.keys()): cat_map = _category_a1(by_lm, llm, _PRIMARY_MODE) parts.append(f"") for cat in categories: mean, n = cat_map.get(cat, (0.0, 0)) parts.append( f'' if n else "" ) parts.append("") parts.append("
LLM{esc(cat)}
{esc(llm)}{mean:.2f}
n={n}		—
") parts.append( "

Paper Fig. 3: Startup/Runtime easy (A@1 > 0.65), " "Admission/Performance hard (A@1 < 0.36). Losing only where the " "paper loses = corpus difficulty; losing broadly = opensre.

" ) return parts def _render_l0_investigation_markdown( by_lm: dict[str, dict[str, list[dict[str, Any]]]], ) -> list[str]: """L0 panel: investigation-native metrics from opensre prose (not predictor).""" if not by_lm: return [] flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs] present = [m for m in _L0_INVESTIGATION_METRICS if _per_cell_metric(flat, m)] if not present: return [] lines: list[str] = [] lines.append("### Investigation quality — L0 (opensre prose, not paper-comparable)") lines.append("") lines.append( "_L0 scores a keyword-parsed triple from opensre's investigation prose " "(``report`` / ``root_cause`` / causal chain). L1 ``a1`` in the headline " "scores the predictor's rank-1 formalization. The gap " "``a1 − investigation_a1`` is translation loss; " "``translation_loss`` flags cases where L0 is right but L1 is wrong._" ) lines.append("") header = "| LLM | variant | " + " | ".join(present) + " |" sep = "|" + "|".join(["---"] * (len(present) + 2)) + "|" lines.append(header) lines.append(sep) for llm in sorted(by_lm.keys()): for mode in sorted(by_lm[llm].keys()): mode_cells = by_lm[llm][mode] row = [f"`{llm}`", mode] for metric in present: scen_vals = _scenario_means(mode_cells, metric) mean, lo, hi, n = _mean_with_ci(scen_vals) if metric in _L0_CI_METRICS and len(scen_vals) >= 2: row.append(f"{mean:.2f} [{lo:.2f}–{hi:.2f}]") elif n: row.append(f"{mean:.2f}") else: row.append("—") lines.append("| " + " | ".join(row) + " |") lines.append("") return lines def _render_l0_investigation_html( by_lm: dict[str, dict[str, list[dict[str, Any]]]], esc: Any, ) -> list[str]: """HTML mirror of :func:`_render_l0_investigation_markdown`.""" if not by_lm: return [] flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs] present = [m for m in _L0_INVESTIGATION_METRICS if _per_cell_metric(flat, m)] if not present: return [] parts: list[str] = [] parts.append("

Investigation quality — L0 (opensre prose, not paper-comparable)

") parts.append( "

L0 scores a keyword-parsed triple from opensre's investigation " "prose. L1 a1 scores the predictor's rank-1 formalization. " "The gap a1 − investigation_a1 is translation loss.

" ) parts.append("") for m in present: parts.append(f"") parts.append("") for llm in sorted(by_lm.keys()): for mode in sorted(by_lm[llm].keys()): mode_cells = by_lm[llm][mode] parts.append(f"") for metric in present: scen_vals = _scenario_means(mode_cells, metric) mean, lo, hi, n = _mean_with_ci(scen_vals) if metric in _L0_CI_METRICS and len(scen_vals) >= 2: cell_txt = f"{mean:.2f}
[{lo:.2f}–{hi:.2f}]" elif n: cell_txt = f"{mean:.2f}" else: cell_txt = "—" parts.append(f'') parts.append("") parts.append("
LLMvariant{esc(m)}
{esc(llm)}{esc(mode)}{cell_txt}
") return parts # --------------------------------------------------------------------------- # # Markdown rendering # # --------------------------------------------------------------------------- # def _render_markdown( report: dict[str, Any], cells: list[dict[str, Any]], provenance: dict[str, Any] | None = None, ) -> str: """Render the report as plain CommonMark.""" lines: list[str] = [] lines.append(f"# Benchmark Run — {report.get('run_id', '(unknown)')}") lines.append("") lines.append( f"_config hash:_ `{report.get('config_hash', '?')}` · " f"_opensre SHA:_ `{report.get('opensre_sha', '?')}`" ) lines.append("") lines.append(f"**Started:** {report.get('started_at', '?')} ") lines.append(f"**Ended:** {report.get('ended_at', '?')} ") cost = report.get("cost", {}) lines.append( f"**Cost:** ${cost.get('total_cost_usd', 0):.4f} of " f"${cost.get('budget_usd', 0):.2f} budget " f"({cost.get('total_calls', 0)} calls, " f"{cost.get('total_tokens_in', 0):,} in / {cost.get('total_tokens_out', 0):,} out)" ) lines.append("") # --- Provenance (Mechanism 5: reproducibility) --- if provenance is not None: lines.extend(_render_provenance_markdown(provenance)) # --- COI disclosure (Mechanism 10) --- coi = (report.get("coi_disclosure") or "").strip() if coi: lines.append("## Conflict-of-interest disclosure") lines.append("") for paragraph in coi.split("\n\n"): lines.append(paragraph.strip()) lines.append("") # --- Headline panel (paper-comparable: per-LLM MEAN + clustered CI) --- lines.append("## Headline — mean per scenario, single-shot (paper-comparable)") lines.append("") lines.append( "Point estimates are **means**, the same aggregation the paper uses " "(A@k is a per-case mean, Wang et al. 2026 §4.2.1). CIs are 95% " "scenario-clustered bootstrap intervals — the independent unit is the " "scenario, not the seed. The `paper` row is the published **Base** " "(zero-shot) baseline over the **full 452-case** corpus (Table 4). A " "head-to-head claim is only valid when our run is also single-shot and " "full-corpus; if the CI overlaps the paper value, the two are " "statistically indistinguishable." ) lines.append("") by_lm = _cells_by_llm_mode(cells) if not by_lm: lines.append("_no cells executed_") else: header = "| LLM | variant | n | " + " | ".join(_PAPER_COMPARABLE_METRICS) + " |" sep = "|" + "|".join(["---"] * (len(_PAPER_COMPARABLE_METRICS) + 3)) + "|" lines.append(header) lines.append(sep) for llm in sorted(by_lm.keys()): for mode in sorted(by_lm[llm].keys()): mode_cells = by_lm[llm][mode] n_scen = len({c.get("case", {}).get("case_id", "?") for c in mode_cells}) row = [f"`{llm}`", mode, str(n_scen)] for metric in _PAPER_COMPARABLE_METRICS: scen_vals = _scenario_means(mode_cells, metric) mean, lo, hi, _ = _mean_with_ci(scen_vals) if metric in ("a1", "a3") and len(scen_vals) >= 2: row.append(f"{mean:.2f} [{lo:.2f}–{hi:.2f}]") else: row.append(f"{mean:.2f}") lines.append("| " + " | ".join(row) + " |") baseline = _match_paper_baseline(llm) if baseline is not None: prow = [f"`{llm}`", "paper-Base", "452"] for metric in _PAPER_COMPARABLE_METRICS: val = baseline.get(metric) prow.append(f"{val:.2f}" if isinstance(val, (int, float)) else "—") lines.append("| " + " | ".join(prow) + " |") icl = _match_paper_icl(llm) if icl is not None: irow = [f"`{llm}`", "paper-ICL", "452"] for metric in _PAPER_COMPARABLE_METRICS: val = icl.get(metric) irow.append(f"{val:.2f}" if isinstance(val, (int, float)) else "—") lines.append("| " + " | ".join(irow) + " |") lines.append("") lines.append( "_`opensre+llm` is the primary arm; `llm_alone` is the same-model " "control. `paper-Base` = zero-shot agent (Table 4); `paper-ICL` = 3 " "retrieved in-context traces, **no agent framework** (Table 5) — the " "cost-equivalent baseline opensre must beat. ICL exists only for the " "three models the paper ran it on._" ) lines.append("") lines.append( "_Excluded from the comparison: " + "; ".join(f"**{m}** ({why})" for m, why in _NON_COMPARABLE_METRICS.items()) + "._" ) # --- Decomposition: where the accuracy goes (Track 2) --- lines.extend(_render_decomposition_markdown(cells, by_lm)) # --- L0 investigation quality (opensre prose — not paper-comparable) --- lines.extend(_render_l0_investigation_markdown(by_lm)) # --- opensre-only diagnostics (NOT in the paper, NOT comparable) --- if by_lm: flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs] present = [m for m in _OPENSRE_ONLY_METRICS if _per_cell_metric(flat, m)] if present: lines.append("### opensre-only diagnostics (not in the paper — do not compare)") lines.append("") lines.append( "_These metrics are opensre instrumentation with no published " "counterpart. `partial_*` relaxes the triple match; `object_*` " "scores component localization alone; the `*_rate` metrics are " "heuristic validity probes. Means shown for internal tracking only._" ) lines.append("") header = "| LLM | variant | " + " | ".join(present) + " |" sep = "|" + "|".join(["---"] * (len(present) + 2)) + "|" lines.append(header) lines.append(sep) for llm in sorted(by_lm.keys()): for mode in sorted(by_lm[llm].keys()): mode_cells = by_lm[llm][mode] row = [f"`{llm}`", mode] for metric in present: scen_vals = _scenario_means(mode_cells, metric) mean, _, _, n = _mean_with_ci(scen_vals) row.append(f"{mean:.2f}" if n else "—") lines.append("| " + " | ".join(row) + " |") lines.append("") # --- Per-stratum × per-LLM detail (Mechanism 4) --- lines.append("## Per-stratum × per-LLM (medians — distributional view)") lines.append("") lines.append( "These are **medians** across seeds (a robustness cross-check, not the " "headline). Stratum semantics:\n" "- `all` / `seen-shape` / `unseen-shape` / `held-out` / `optimize`: " "single-shot strata — each seed is one independent draw.\n" "- `consistency-selected`: **best-of-N** — the adapter picks the most " "self-consistent of the repeated runs per scenario. This is an " "*optimistic* selection and is **NOT comparable** to the paper's " "single-shot Table 4 baselines; report it separately and never as the " "headline." ) lines.append("") reported_metrics = report.get("reported_metrics", []) per_stratum = report.get("per_stratum", {}) for stratum in sorted(per_stratum.keys()): label = ( " — best-of-N, optimistic, not paper-comparable" if stratum == "consistency-selected" else "" ) lines.append(f"### {stratum}{label}") lines.append("") by_mode_llm = per_stratum[stratum] if not by_mode_llm: lines.append("_no data_") lines.append("") continue header = "| mode/llm | " + " | ".join(reported_metrics) + " |" sep = "|" + "|".join(["---"] * (len(reported_metrics) + 1)) + "|" lines.append(header) lines.append(sep) for mode_llm in sorted(by_mode_llm.keys()): metrics = by_mode_llm[mode_llm] row = [f"`{mode_llm}`"] for metric in reported_metrics: value = metrics.get(metric, 0.0) row.append(f"{value:.2f}" if isinstance(value, (int, float)) else "—") lines.append("| " + " | ".join(row) + " |") lines.append("") # --- Negative results section (Mechanism 9) --- lines.append("## Negative results — where opensre lost or tied") lines.append("") negative = (report.get("negative_results") or "").strip() lines.append("```") lines.append(negative or "(none recorded)") lines.append("```") lines.append("") # --- Pre-registration pointer (Mechanism 1) --- prereg = report.get("pre_registration_path") if prereg: lines.append("## Pre-registration") lines.append("") lines.append(f"`{prereg}` (committed before run; expected deltas were locked in)") lines.append("") # --- Raw artifacts (Mechanism 5) --- raw_dir = report.get("raw_artifacts_dir") if raw_dir: lines.append("## Raw artifacts") lines.append("") lines.append(f"Per-case JSON written to `{raw_dir}` ({len(cells)} files).") lines.append("") # --- Cost breakdown by model --- by_model = cost.get("by_model", {}) if by_model: lines.append("## Cost breakdown by model") lines.append("") lines.append("| model | calls | tokens in | tokens out | cost USD |") lines.append("|---|---|---|---|---|") for model in sorted(by_model.keys()): m = by_model[model] lines.append( f"| `{model}` | {m.get('call_count', 0)} | " f"{m.get('tokens_in', 0):,} | {m.get('tokens_out', 0):,} | " f"${m.get('cost_usd', 0):.4f} |" ) lines.append("") return "\n".join(lines) + "\n" # --------------------------------------------------------------------------- # # HTML rendering — self-contained, inline CSS, no external assets # # --------------------------------------------------------------------------- # _HTML_STYLE = """ :root { --fg: #1a1a1a; --bg: #ffffff; --muted: #5a6172; --soft: #f5f7fa; --line: #e1e4e8; --accent: #0066cc; --good: #1a7f4f; --warn: #b85c00; --bad: #b91c1c; --shadow: 0 1px 3px rgba(0,0,0,0.05); } * { box-sizing: border-box; } body { margin: 0; padding: 2rem; max-width: 1200px; margin: 0 auto; font-family: -apple-system, BlinkMacSystemFont, "Inter", sans-serif; color: var(--fg); background: var(--bg); line-height: 1.5; font-size: 14px; } h1 { margin: 0 0 0.5rem 0; font-size: 1.8rem; } h2 { font-size: 1.25rem; margin: 2rem 0 0.75rem 0; border-bottom: 2px solid var(--accent); padding-bottom: 0.3rem; } h3 { font-size: 1rem; margin: 1.25rem 0 0.5rem 0; color: var(--muted); } .meta { display: grid; grid-template-columns: max-content 1fr; gap: 0.25rem 1rem; font-size: 13px; color: var(--muted); margin-bottom: 1rem; } .meta dt { font-weight: 600; color: var(--fg); } table { border-collapse: collapse; width: 100%; margin: 0.5rem 0; font-size: 13px; background: white; box-shadow: var(--shadow); border-radius: 6px; overflow: hidden; } th, td { text-align: left; padding: 0.5rem 0.75rem; border-bottom: 1px solid var(--line); } th { background: var(--soft); font-weight: 600; font-size: 11px; text-transform: uppercase; letter-spacing: 0.04em; } tbody tr:last-child td { border-bottom: none; } tbody tr:hover { background: var(--soft); } td.metric { font-variant-numeric: tabular-nums; text-align: right; } .pill { display: inline-block; padding: 1px 8px; border-radius: 12px; font-size: 11px; font-weight: 600; background: #e6f0ff; color: var(--accent); } .pill.good { background: #e8f5ee; color: var(--good); } .pill.warn { background: #fff4e6; color: var(--warn); } .pill.bad { background: #fee2e2; color: var(--bad); } pre { background: var(--soft); border: 1px solid var(--line); border-radius: 6px; padding: 0.75rem; overflow-x: auto; font-size: 12px; } code { font-family: "SF Mono", Monaco, Menlo, Consolas, monospace; font-size: 0.9em; } .callout { border-left: 4px solid var(--accent); background: #f4f8ff; padding: 0.6rem 1rem; margin: 1rem 0; border-radius: 0 6px 6px 0; } .callout.coi { border-left-color: var(--warn); background: #fff8ec; } """ def _render_html( report: dict[str, Any], cells: list[dict[str, Any]], provenance: dict[str, Any] | None = None, ) -> str: """Render a self-contained HTML report. No external CSS or JS.""" def esc(s: Any) -> str: return html.escape(str(s)) parts: list[str] = [] parts.append("") parts.append('') parts.append('') parts.append('') parts.append(f"Benchmark Run — {esc(report.get('run_id', ''))}") parts.append(f"") parts.append("") # Title + meta parts.append(f"

Benchmark Run — {esc(report.get('run_id', '(unknown)'))}

") parts.append('
') parts.append(f"
Config hash
{esc(report.get('config_hash', '?'))}
") parts.append(f"
opensre SHA
{esc(report.get('opensre_sha', '?'))}
") parts.append(f"
Started
{esc(report.get('started_at', '?'))}
") parts.append(f"
Ended
{esc(report.get('ended_at', '?'))}
") cost = report.get("cost", {}) parts.append( f"
Cost
${cost.get('total_cost_usd', 0):.4f} of " f"${cost.get('budget_usd', 0):.2f} budget " f"({cost.get('total_calls', 0)} calls)
" ) parts.append("
") # Provenance section (Mechanism 5) if provenance is not None: parts.extend(_render_provenance_html(provenance, esc)) # COI coi = (report.get("coi_disclosure") or "").strip() if coi: parts.append("

Conflict-of-interest disclosure

") parts.append('
') for paragraph in coi.split("\n\n"): parts.append(f"

{esc(paragraph.strip())}

") parts.append("
") # Headline panel — paper-comparable mean + scenario-clustered CI parts.append("

Headline — mean per scenario, single-shot (paper-comparable)

") parts.append( '

Point estimates are means ' "(matching the paper, where A@k is a per-case mean). CIs are 95% " "scenario-clustered bootstrap intervals. The paper row is " "the published Base baseline over the full 452-case " "corpus (Wang et al. 2026, Table 4). A head-to-head claim is only valid " "when our run is single-shot and full-corpus; if the CI overlaps the " "paper value, the two are statistically indistinguishable.

" ) by_lm = _cells_by_llm_mode(cells) if not by_lm: parts.append("

no cells executed

") else: parts.append("") for m in _PAPER_COMPARABLE_METRICS: parts.append(f"") parts.append("") for llm in sorted(by_lm.keys()): for mode in sorted(by_lm[llm].keys()): mode_cells = by_lm[llm][mode] n_scen = len({c.get("case", {}).get("case_id", "?") for c in mode_cells}) parts.append( f"" f'' ) for m in _PAPER_COMPARABLE_METRICS: scen_vals = _scenario_means(mode_cells, m) mean, lo, hi, _ = _mean_with_ci(scen_vals) if m in ("a1", "a3") and len(scen_vals) >= 2: cell_txt = f"{mean:.2f}
[{lo:.2f}–{hi:.2f}]" else: cell_txt = f"{mean:.2f}" parts.append(f'') parts.append("") baseline = _match_paper_baseline(llm) if baseline is not None: parts.append( f"" '' '' ) for m in _PAPER_COMPARABLE_METRICS: val = baseline.get(m) txt = f"{val:.2f}" if isinstance(val, (int, float)) else "—" parts.append(f'') parts.append("") icl = _match_paper_icl(llm) if icl is not None: parts.append( f"" '' '' ) for m in _PAPER_COMPARABLE_METRICS: val = icl.get(m) txt = f"{val:.2f}" if isinstance(val, (int, float)) else "—" parts.append(f'') parts.append("") parts.append("
LLMvariantn{esc(m)}
{esc(llm)}{esc(mode)}{n_scen}{cell_txt}
{esc(llm)}paper-Base452{txt}
{esc(llm)}paper-ICL452{txt}
") parts.append( "

opensre+llm is the primary arm; " "llm_alone is the same-model control. " "paper-Base = zero-shot agent (Table 4); " "paper-ICL = 3 retrieved in-context traces, no " "agent framework (Table 5) — the cost-equivalent baseline " "opensre must beat. ICL exists only for the three models the paper " "ran it on.

" ) excluded = "; ".join( f"{esc(m)} ({esc(why)})" for m, why in _NON_COMPARABLE_METRICS.items() ) parts.append(f"

Excluded from the comparison: {excluded}.

") # Decomposition (Track 2) parts.extend(_render_decomposition_html(cells, by_lm, esc)) # L0 investigation quality parts.extend(_render_l0_investigation_html(by_lm, esc)) # opensre-only diagnostics (segregated — not paper-comparable) flat = [c for modes in by_lm.values() for cs in modes.values() for c in cs] present = [m for m in _OPENSRE_ONLY_METRICS if _per_cell_metric(flat, m)] if present: parts.append("

opensre-only diagnostics (not in the paper — do not compare)

") parts.append("") for m in present: parts.append(f"") parts.append("") for llm in sorted(by_lm.keys()): for mode in sorted(by_lm[llm].keys()): mode_cells = by_lm[llm][mode] parts.append(f"") for m in present: scen_vals = _scenario_means(mode_cells, m) mean, _, _, n = _mean_with_ci(scen_vals) parts.append(f'' if n else "") parts.append("") parts.append("
LLMvariant{esc(m)}
{esc(llm)}{esc(mode)}{mean:.2f}—
") # Per-stratum × per-LLM parts.append("

Per-stratum × per-LLM (medians — distributional view)

") parts.append( '

These are medians across ' "seeds (a robustness cross-check, not the headline). " "all/seen-shape/unseen-shape/" "held-out/optimize are single-shot strata. " "consistency-selected is best-of-N — an " "optimistic selection that is not comparable to the " "paper's single-shot baselines.

" ) reported_metrics = report.get("reported_metrics", []) for stratum in sorted(report.get("per_stratum", {}).keys()): label = ( " — best-of-N, optimistic, not paper-comparable" if stratum == "consistency-selected" else "" ) parts.append(f"

{esc(stratum)}{esc(label)}

") by_mode_llm = report["per_stratum"][stratum] if not by_mode_llm: parts.append("

no data

") continue parts.append("") for m in reported_metrics: parts.append(f"") parts.append("") for mode_llm in sorted(by_mode_llm.keys()): metrics = by_mode_llm[mode_llm] parts.append(f"") for m in reported_metrics: value = metrics.get(m, 0.0) cell = f"{value:.2f}" if isinstance(value, (int, float)) else "—" parts.append(f'') parts.append("") parts.append("
mode/llm{esc(m)}
{esc(mode_llm)}{cell}
") # Negative results parts.append("

Negative results — where opensre lost or tied

") negative = (report.get("negative_results") or "").strip() parts.append(f"
{esc(negative or '(none recorded)')}
") # Pre-registration prereg = report.get("pre_registration_path") if prereg: parts.append("

Pre-registration

") parts.append( f"

{esc(prereg)} — committed before run; " "expected deltas were locked in.

" ) # Raw artifacts raw_dir = report.get("raw_artifacts_dir") if raw_dir: parts.append("

Raw artifacts

") parts.append( f"

Per-case JSON written to {esc(raw_dir)} ({len(cells)} files).

" ) # Cost breakdown by_model = cost.get("by_model", {}) if by_model: parts.append("

Cost breakdown by model

") parts.append( "" "" ) for model in sorted(by_model.keys()): m = by_model[model] parts.append( f"" f'' f'' f'' f'' ) parts.append("
modelcallstokens intokens outcost USD
{esc(model)}{m.get("call_count", 0)}{m.get("tokens_in", 0):,}{m.get("tokens_out", 0):,}${m.get("cost_usd", 0):.4f}
") parts.append("") return "\n".join(parts) + "\n" # --------------------------------------------------------------------------- # # Provenance renderers — surface "what exact code + config + env ran" # # --------------------------------------------------------------------------- # def _render_provenance_markdown(prov: dict[str, Any]) -> list[str]: """Markdown section with the highest-leverage provenance fields. Full content (config YAML, pre-reg YAML, full env) stays in ``provenance.json`` — the report just summarizes so reviewers know what to look for. Keep this short. """ lines: list[str] = [] code = prov.get("code", {}) env = prov.get("environment", {}) dataset = prov.get("dataset", {}) config_section = prov.get("config", {}) pre_reg = prov.get("pre_registration", {}) lines.append("## Provenance (Mechanism 5: reproducibility)") lines.append("") dirty_marker = " **(DIRTY — uncommitted changes)**" if code.get("opensre_dirty") else "" lines.append( f"- **Code**: `{code.get('opensre_short_sha', '?')}` on " f"`{code.get('opensre_branch', '?')}`{dirty_marker}" ) if code.get("opensre_dirty") and code.get("opensre_changed_files"): changed = code["opensre_changed_files"] files_str = ", ".join(f"`{f}`" for f in changed[:5]) suffix = f" (+{len(changed) - 5} more)" if len(changed) > 5 else "" lines.append(f" - Changed files: {files_str}{suffix}") if config_section.get("path"): lines.append( f"- **Config**: `{config_section['path']}` " f"(sha256 `{(config_section.get('sha256') or '?')[:12]}…`)" ) if pre_reg.get("path"): lines.append( f"- **Pre-registration**: `{pre_reg['path']}` " f"(sha256 `{(pre_reg.get('sha256') or '?')[:12]}…`)" ) if dataset.get("hf_dataset"): rev = dataset.get("hf_revision") or "(unpinned)" lines.append(f"- **Dataset**: {dataset['hf_dataset']} @ `{rev}`") lines.append( f"- **Python**: {env.get('python_version', '?')} " f"({env.get('python_implementation', '?')}) on {env.get('platform', '?')}" ) key_packages = env.get("key_packages", {}) if key_packages: pkg_str = ", ".join( f"{name} {version}" for name, version in sorted(key_packages.items()) if version ) if pkg_str: lines.append(f"- **Key packages**: {pkg_str}") lines.append("") lines.append( "_Full provenance — config + pre-registration contents, every package " "version, allowlisted env vars — lives in `provenance.json` in this " "run directory._" ) lines.append("") return lines def _render_provenance_html(prov: dict[str, Any], esc: Any) -> list[str]: code = prov.get("code", {}) env = prov.get("environment", {}) dataset = prov.get("dataset", {}) config_section = prov.get("config", {}) pre_reg = prov.get("pre_registration", {}) parts: list[str] = [] parts.append("

Provenance (Mechanism 5: reproducibility)

") parts.append('
') dirty_pill = ' DIRTY' if code.get("opensre_dirty") else "" parts.append( f"
Code
{esc(code.get('opensre_short_sha', '?'))} " f"on {esc(code.get('opensre_branch', '?'))}{dirty_pill}
" ) if code.get("opensre_dirty") and code.get("opensre_changed_files"): changed = code["opensre_changed_files"] files_html = ", ".join(f"{esc(f)}" for f in changed[:5]) suffix = f" (+{len(changed) - 5} more)" if len(changed) > 5 else "" parts.append(f"
Changed files
{files_html}{esc(suffix)}
") if config_section.get("path"): sha = (config_section.get("sha256") or "?")[:12] parts.append( f"
Config
{esc(config_section['path'])} " f"(sha256 {esc(sha)}…)
" ) if pre_reg.get("path"): sha = (pre_reg.get("sha256") or "?")[:12] parts.append( f"
Pre-registration
{esc(pre_reg['path'])} " f"(sha256 {esc(sha)}…)
" ) if dataset.get("hf_dataset"): rev = dataset.get("hf_revision") or "(unpinned)" parts.append( f"
Dataset
{esc(dataset['hf_dataset'])} @ {esc(rev)}
" ) parts.append( f"
Python
{esc(env.get('python_version', '?'))} " f"({esc(env.get('python_implementation', '?'))}) on " f"{esc(env.get('platform', '?'))}
" ) key_packages = env.get("key_packages", {}) pkg_items = sorted((n, v) for n, v in key_packages.items() if v) if pkg_items: pkg_str = ", ".join(f"{esc(n)} {esc(v)}" for n, v in pkg_items) parts.append(f"
Key packages
{pkg_str}
") parts.append("
") parts.append( "

Full provenance — config + pre-registration contents, " "every package version, allowlisted env vars — lives in " "provenance.json in this run directory.

" ) return parts