wehub-resource-sync
90 lines
2.8 KiB
Python
90 lines
2.8 KiB
Python
"""Benchmark report schema + metadata capture.
|
|
|
|
:func:`build_report` assembles the single JSON document the harness writes.
|
|
Its per-journey ``summary`` + ``runs`` shape mirrors MLflow's gateway
|
|
benchmark so the workspace ETL notebook flattens it unchanged — keyed by
|
|
journey (and ``harness``) instead of ``backend``. Bump :data:`SCHEMA_VERSION`
|
|
whenever the document's shape changes so the ETL can branch on it.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import platform
|
|
import subprocess
|
|
|
|
# Incremented on any breaking change to the report document shape below.
|
|
SCHEMA_VERSION = 1
|
|
|
|
|
|
def _git(*args: str) -> str:
|
|
"""Run ``git *args`` at the repo root, returning stripped stdout or ``""``.
|
|
|
|
Never raises: a missing git, detached checkout, or non-zero exit all
|
|
surface as an empty string so a benchmark run outside a clean checkout
|
|
still produces a valid report.
|
|
"""
|
|
try:
|
|
out = subprocess.run(
|
|
["git", *args],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
check=False,
|
|
)
|
|
except (OSError, subprocess.SubprocessError):
|
|
return ""
|
|
return out.stdout.strip() if out.returncode == 0 else ""
|
|
|
|
|
|
def git_sha() -> str:
|
|
"""Return the current commit SHA, or ``""`` when unavailable."""
|
|
return _git("rev-parse", "HEAD")
|
|
|
|
|
|
def git_branch() -> str:
|
|
"""Return the current branch name, or ``""`` when detached/unavailable."""
|
|
return _git("rev-parse", "--abbrev-ref", "HEAD")
|
|
|
|
|
|
def host_info() -> dict[str, object]:
|
|
"""Capture coarse host facts for cross-machine result comparison."""
|
|
import os
|
|
|
|
return {
|
|
"platform": platform.platform(),
|
|
"python": platform.python_version(),
|
|
"cpu_count": os.cpu_count(),
|
|
}
|
|
|
|
|
|
def build_report(
|
|
journey_results: dict[str, dict[str, object]],
|
|
*,
|
|
generated_at: str,
|
|
config: dict[str, object],
|
|
harness: str,
|
|
) -> dict[str, object]:
|
|
"""Assemble the full benchmark report document.
|
|
|
|
:param journey_results: Per-journey ``{"kind", "runs", "summary"}``
|
|
blocks (each ``runs``/``summary`` produced by
|
|
:func:`measure.aggregate`), keyed by journey name.
|
|
:param generated_at: ISO-8601 timestamp stamped by the caller (kept out
|
|
of this pure function so it stays deterministic under test).
|
|
:param config: The run's knobs (iterations, requests, concurrency, runs,
|
|
mock_llm) for provenance.
|
|
:param harness: Harness driving full-turn journeys, e.g.
|
|
``"openai-agents"``.
|
|
:returns: The JSON-serializable report document.
|
|
"""
|
|
return {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"generated_at": generated_at,
|
|
"git_sha": git_sha(),
|
|
"git_branch": git_branch(),
|
|
"host": host_info(),
|
|
"harness": harness,
|
|
"config": config,
|
|
"journeys": journey_results,
|
|
}
|