"""Benchmark report schema + metadata capture. :func:`build_report` assembles the single JSON document the harness writes. Its per-journey ``summary`` + ``runs`` shape mirrors MLflow's gateway benchmark so the workspace ETL notebook flattens it unchanged — keyed by journey (and ``harness``) instead of ``backend``. Bump :data:`SCHEMA_VERSION` whenever the document's shape changes so the ETL can branch on it. """ from __future__ import annotations import platform import subprocess # Incremented on any breaking change to the report document shape below. SCHEMA_VERSION = 1 def _git(*args: str) -> str: """Run ``git *args`` at the repo root, returning stripped stdout or ``""``. Never raises: a missing git, detached checkout, or non-zero exit all surface as an empty string so a benchmark run outside a clean checkout still produces a valid report. """ try: out = subprocess.run( ["git", *args], capture_output=True, text=True, timeout=10, check=False, ) except (OSError, subprocess.SubprocessError): return "" return out.stdout.strip() if out.returncode == 0 else "" def git_sha() -> str: """Return the current commit SHA, or ``""`` when unavailable.""" return _git("rev-parse", "HEAD") def git_branch() -> str: """Return the current branch name, or ``""`` when detached/unavailable.""" return _git("rev-parse", "--abbrev-ref", "HEAD") def host_info() -> dict[str, object]: """Capture coarse host facts for cross-machine result comparison.""" import os return { "platform": platform.platform(), "python": platform.python_version(), "cpu_count": os.cpu_count(), } def build_report( journey_results: dict[str, dict[str, object]], *, generated_at: str, config: dict[str, object], harness: str, ) -> dict[str, object]: """Assemble the full benchmark report document. :param journey_results: Per-journey ``{"kind", "runs", "summary"}`` blocks (each ``runs``/``summary`` produced by :func:`measure.aggregate`), keyed by journey name. :param generated_at: ISO-8601 timestamp stamped by the caller (kept out of this pure function so it stays deterministic under test). :param config: The run's knobs (iterations, requests, concurrency, runs, mock_llm) for provenance. :param harness: Harness driving full-turn journeys, e.g. ``"openai-agents"``. :returns: The JSON-serializable report document. """ return { "schema_version": SCHEMA_VERSION, "generated_at": generated_at, "git_sha": git_sha(), "git_branch": git_branch(), "host": host_info(), "harness": harness, "config": config, "journeys": journey_results, }