yao-meta-skill/scripts/render_benchmark_reproducibility.py

#!/usr/bin/env python3
import argparse
import hashlib
import json
import subprocess
from datetime import date
from pathlib import Path
from typing import Any


ROOT = Path(__file__).resolve().parent.parent
SCRIPT_INTERFACE = "cli"
SCRIPT_INTERFACE_REASON = "Renders a release-facing benchmark reproducibility manifest and Markdown report."

METHODOLOGY_SECTIONS = [
    "## Benchmark Types",
    "## Sample Sources",
    "## Evaluation Dimensions",
    "## Weighting Rule",
    "## Failure Disclosure",
    "## Reproduction",
]

REQUIRED_ARTIFACTS = [
    ("methodology", "reports/benchmark_methodology.md"),
    ("failure_disclosure", "evals/failure-cases.md"),
    ("output_cases", "evals/output/cases.jsonl"),
    ("output_schema", "evals/output/schema.json"),
    ("output_scorecard", "reports/output_quality_scorecard.json"),
    ("output_execution", "reports/output_execution_runs.json"),
    ("blind_review", "reports/output_blind_review_pack.json"),
    ("review_adjudication", "reports/output_review_adjudication.json"),
    ("trigger_scorecard", "reports/route_scorecard.json"),
    ("runtime_conformance", "reports/conformance_matrix.json"),
    ("trust_report", "reports/security_trust_report.json"),
    ("python_compatibility", "reports/python_compatibility.json"),
    ("registry_audit", "reports/registry_audit.json"),
    ("package_verification", "reports/package_verification.json"),
    ("install_simulation", "reports/install_simulation.json"),
    ("skill_os2_audit", "reports/skill_os2_audit.json"),
    ("world_class_evidence_plan", "reports/world_class_evidence_plan.json"),
    ("world_class_evidence_ledger", "reports/world_class_evidence_ledger.json"),
    ("world_class_evidence_intake", "reports/world_class_evidence_intake.json"),
    ("world_class_submission_review", "reports/world_class_submission_review.json"),
    ("world_class_operator_runbook", "reports/world_class_operator_runbook.json"),
    ("world_class_operator_runbook_markdown", "reports/world_class_operator_runbook.md"),
    ("world_class_operator_runbook_html", "reports/world_class_operator_runbook.html"),
    ("world_class_claim_guard", "reports/world_class_claim_guard.json"),
]

REPRODUCTION_COMMANDS = [
    {
        "label": "source commit",
        "command": "git rev-parse HEAD",
        "evidence": "git commit hash",
    },
    {
        "label": "trigger eval",
        "command": "make eval-suite",
        "evidence": "reports/eval_suite.json",
    },
    {
        "label": "output eval",
        "command": "python3 scripts/yao.py output-eval",
        "evidence": "reports/output_quality_scorecard.json",
    },
    {
        "label": "output execution",
        "command": "python3 scripts/yao.py output-exec --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'",
        "evidence": "reports/output_execution_runs.json",
    },
    {
        "label": "blind review adjudication",
        "command": "python3 scripts/yao.py output-review",
        "evidence": "reports/output_review_adjudication.json",
    },
    {
        "label": "skill ir",
        "command": "python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json",
        "evidence": "skill-ir/examples/yao-meta-skill.json",
    },
    {
        "label": "runtime conformance",
        "command": "python3 scripts/yao.py conformance .",
        "evidence": "reports/conformance_matrix.json",
    },
    {
        "label": "trust report",
        "command": "python3 scripts/yao.py trust .",
        "evidence": "reports/security_trust_report.json",
    },
    {
        "label": "python compatibility",
        "command": "python3 scripts/yao.py python-compat .",
        "evidence": "reports/python_compatibility.json",
    },
    {
        "label": "package",
        "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip",
        "evidence": "dist/yao-meta-skill.zip",
    },
    {
        "label": "package verify",
        "command": "python3 scripts/yao.py package-verify . --package-dir dist --require-zip",
        "evidence": "reports/package_verification.json",
    },
    {
        "label": "install simulate",
        "command": "python3 scripts/yao.py install-simulate . --package-dir dist",
        "evidence": "reports/install_simulation.json",
    },
    {
        "label": "registry audit",
        "command": "python3 scripts/yao.py registry-audit .",
        "evidence": "reports/registry_audit.json",
    },
    {
        "label": "skill os audit",
        "command": "python3 scripts/yao.py skill-os2-audit .",
        "evidence": "reports/skill_os2_audit.json",
    },
    {
        "label": "world-class evidence plan",
        "command": "python3 scripts/yao.py world-class-evidence .",
        "evidence": "reports/world_class_evidence_plan.json",
    },
    {
        "label": "world-class evidence ledger",
        "command": "python3 scripts/yao.py world-class-ledger .",
        "evidence": "reports/world_class_evidence_ledger.json",
    },
    {
        "label": "world-class evidence intake",
        "command": "python3 scripts/yao.py world-class-intake .",
        "evidence": "reports/world_class_evidence_intake.json",
    },
    {
        "label": "world-class submission review",
        "command": "python3 scripts/yao.py world-class-submission-review .",
        "evidence": "reports/world_class_submission_review.json",
    },
    {
        "label": "world-class operator runbook",
        "command": "python3 scripts/yao.py world-class-runbook .",
        "evidence": "reports/world_class_operator_runbook.json",
    },
    {
        "label": "world-class claim guard",
        "command": "python3 scripts/yao.py world-class-claim-guard .",
        "evidence": "reports/world_class_claim_guard.json",
    },
    {
        "label": "full ci",
        "command": "make ci-test",
        "evidence": "CI target output",
    },
]


def load_json(path: Path) -> dict[str, Any]:
    if not path.exists():
        return {}
    try:
        payload = json.loads(path.read_text(encoding="utf-8"))
    except json.JSONDecodeError:
        return {}
    return payload if isinstance(payload, dict) else {}


def rel_path(path: Path, root: Path) -> str:
    try:
        return str(path.resolve().relative_to(root.resolve()))
    except ValueError:
        return str(path.resolve())


def sha256_file(path: Path) -> str:
    digest = hashlib.sha256()
    with path.open("rb") as handle:
        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def git_commit(skill_dir: Path) -> str:
    try:
        proc = subprocess.run(
            ["git", "rev-parse", "HEAD"],
            cwd=skill_dir,
            capture_output=True,
            text=True,
            check=True,
        )
    except (OSError, subprocess.CalledProcessError):
        return "unknown"
    return proc.stdout.strip() or "unknown"


def git_status(skill_dir: Path) -> dict[str, Any]:
    try:
        proc = subprocess.run(
            ["git", "status", "--short", "--untracked-files=all"],
            cwd=skill_dir,
            capture_output=True,
            text=True,
            check=True,
        )
    except (OSError, subprocess.CalledProcessError):
        return {"available": False, "dirty": None, "changed_file_count": None}
    lines = [line for line in proc.stdout.splitlines() if line.strip()]
    return {
        "available": True,
        "dirty": bool(lines),
        "changed_file_count": len(lines),
        "sample": lines[:12],
        "scope": "generation-time status before this report is written",
    }


def count_jsonl(path: Path) -> int:
    if not path.exists():
        return 0
    return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip())


def count_failure_cases(path: Path) -> int:
    if not path.exists():
        return 0
    return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.startswith("### "))


def methodology_check(path: Path) -> dict[str, Any]:
    text = path.read_text(encoding="utf-8") if path.exists() else ""
    sections = [{"heading": heading, "exists": heading in text} for heading in METHODOLOGY_SECTIONS]
    return {
        "path": "reports/benchmark_methodology.md",
        "exists": path.exists(),
        "sections": sections,
        "missing_sections": [item["heading"] for item in sections if not item["exists"]],
    }


def artifact_record(skill_dir: Path, label: str, rel: str) -> dict[str, Any]:
    path = skill_dir / rel
    record: dict[str, Any] = {
        "label": label,
        "path": rel,
        "exists": path.exists(),
    }
    if path.exists() and path.is_file():
        record["bytes"] = path.stat().st_size
        record["sha256"] = sha256_file(path)
    return record


def evidence_bundle_fingerprint(artifacts: list[dict[str, Any]]) -> dict[str, Any]:
    digest = hashlib.sha256()
    existing_count = 0
    missing_paths = []
    for artifact in sorted(artifacts, key=lambda item: str(item.get("path", ""))):
        path = str(artifact.get("path", ""))
        digest.update(path.encode("utf-8"))
        digest.update(b"\0")
        digest.update(str(artifact.get("label", "")).encode("utf-8"))
        digest.update(b"\0")
        digest.update(str(bool(artifact.get("exists"))).encode("utf-8"))
        digest.update(b"\0")
        digest.update(str(artifact.get("sha256", "")).encode("utf-8"))
        digest.update(b"\0")
        if artifact.get("exists"):
            existing_count += 1
        else:
            missing_paths.append(path)
    return {
        "algorithm": "sha256(path,label,exists,artifact_sha256)",
        "artifact_count": len(artifacts),
        "existing_count": existing_count,
        "missing_count": len(missing_paths),
        "missing_paths": missing_paths,
        "sha256": digest.hexdigest(),
    }


def release_lock_status(status: dict[str, Any], commit: str) -> dict[str, Any]:
    available = status.get("available") is True
    clean = status.get("dirty") is False
    known_commit = bool(commit and commit != "unknown")
    ready = available and clean and known_commit
    reasons = []
    if not available:
        reasons.append("git status unavailable")
    if not known_commit:
        reasons.append("git commit unavailable")
    if status.get("dirty") is True:
        reasons.append("working tree was dirty at generation time")
    if status.get("dirty") is None:
        reasons.append("working tree cleanliness unknown")
    if not reasons:
        reasons.append("clean generation-time HEAD")
    return {
        "ready": ready,
        "commit": commit,
        "status_scope": status.get("scope", "generation-time status"),
        "reason": "; ".join(reasons),
    }


def public_claim_blockers(
    local_reproducibility_ready: bool,
    release_lock_ready: bool,
    provider_evidence_complete: bool,
    human_review_complete: bool,
    world_class_ready: bool,
    world_class_open_gap_count: int,
    world_class_ledger_pending_count: int,
) -> list[str]:
    blockers = []
    if not local_reproducibility_ready:
        blockers.append("local benchmark reproducibility is incomplete")
    if not release_lock_ready:
        blockers.append("release lock is not clean or commit is unavailable")
    if not provider_evidence_complete:
        blockers.append("provider-backed model holdout evidence is incomplete")
    if not human_review_complete:
        blockers.append("human blind-review adjudication is incomplete")
    if not world_class_ready:
        blockers.append(
            f"world-class evidence is not accepted yet ({world_class_open_gap_count} open gaps, "
            f"{world_class_ledger_pending_count} ledger pending)"
        )
    return blockers


def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
    reports = skill_dir / "reports"
    output_quality = load_json(reports / "output_quality_scorecard.json")
    output_execution = load_json(reports / "output_execution_runs.json")
    output_review = load_json(reports / "output_review_adjudication.json")
    skill_os2 = load_json(reports / "skill_os2_audit.json")
    world_class_plan = load_json(reports / "world_class_evidence_plan.json")
    world_class_ledger = load_json(reports / "world_class_evidence_ledger.json")
    trust = load_json(reports / "security_trust_report.json")
    package_verification = load_json(reports / "package_verification.json")
    methodology = methodology_check(reports / "benchmark_methodology.md")
    artifacts = [artifact_record(skill_dir, label, rel) for label, rel in REQUIRED_ARTIFACTS]
    evidence_bundle = evidence_bundle_fingerprint(artifacts)
    missing_artifacts = [item["path"] for item in artifacts if not item["exists"]]
    output_summary = output_quality.get("summary", {})
    execution_summary = output_execution.get("summary", {})
    review_summary = output_review.get("summary", {})
    failure_case_count = count_failure_cases(skill_dir / "evals" / "failure-cases.md")
    output_case_count = count_jsonl(skill_dir / "evals" / "output" / "cases.jsonl")
    status = git_status(skill_dir)
    commit = git_commit(skill_dir)
    release_lock = release_lock_status(status, commit)
    local_reproducibility_ready = (
        not methodology["missing_sections"]
        and not missing_artifacts
        and output_case_count >= 5
        and failure_case_count > 0
        and output_summary.get("gate_pass") is True
        and execution_summary.get("command_executed_count", 0) > 0
        and execution_summary.get("timing_observed_count", 0) > 0
    )
    human_review_complete = review_summary.get("pair_count", 0) > 0 and review_summary.get("pending_count", 0) == 0
    provider_evidence_complete = execution_summary.get("model_executed_count", 0) > 0 and execution_summary.get("token_observed_count", 0) > 0
    world_class_ready = bool(skill_os2.get("summary", {}).get("world_class_ready", False))
    world_class_open_gap_count = int(skill_os2.get("summary", {}).get("open_gap_count", 0) or 0)
    world_class_ledger_pending_count = int(world_class_ledger.get("summary", {}).get("pending_count", 0) or 0)
    claim_blockers = public_claim_blockers(
        local_reproducibility_ready,
        release_lock["ready"],
        provider_evidence_complete,
        human_review_complete,
        world_class_ready,
        world_class_open_gap_count,
        world_class_ledger_pending_count,
    )
    public_claim_ready = not claim_blockers
    return {
        "schema_version": "1.0",
        "ok": local_reproducibility_ready,
        "generated_at": generated_at,
        "skill_dir": rel_path(skill_dir, ROOT),
        "commit": commit,
        "git_status": status,
        "summary": {
            "reproducibility_ready": local_reproducibility_ready,
            "release_lock_ready": release_lock["ready"],
            "methodology_complete": not methodology["missing_sections"],
            "required_artifact_count": len(artifacts),
            "missing_artifact_count": len(missing_artifacts),
            "evidence_bundle_sha256": evidence_bundle["sha256"],
            "source_contract_sha256": trust.get("summary", {}).get("package_sha256", ""),
            "archive_sha256": package_verification.get("summary", {}).get("archive_sha256", ""),
            "output_case_count": output_case_count,
            "failure_disclosure_count": failure_case_count,
            "command_count": len(REPRODUCTION_COMMANDS),
            "command_executed_count": execution_summary.get("command_executed_count", 0),
            "timing_observed_count": execution_summary.get("timing_observed_count", 0),
            "model_executed_count": execution_summary.get("model_executed_count", 0),
            "token_observed_count": execution_summary.get("token_observed_count", 0),
            "human_review_complete": human_review_complete,
            "provider_evidence_complete": provider_evidence_complete,
            "world_class_ready": world_class_ready,
            "world_class_open_gap_count": world_class_open_gap_count,
            "world_class_task_count": world_class_plan.get("summary", {}).get("task_count", 0),
            "world_class_ledger_pending_count": world_class_ledger_pending_count,
            "public_claim_ready": public_claim_ready,
            "public_claim_blocker_count": len(claim_blockers),
            "working_tree_dirty": status.get("dirty"),
            "changed_file_count": status.get("changed_file_count"),
        },
        "public_claim": {
            "ready": public_claim_ready,
            "scope": "public benchmark or world-class readiness claim",
            "blockers": claim_blockers,
            "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, and accepted world-class evidence.",
        },
        "release_lock": release_lock,
        "evidence_bundle": evidence_bundle,
        "methodology": methodology,
        "artifacts_checked": artifacts,
        "missing_artifacts": missing_artifacts,
        "reproduction_commands": REPRODUCTION_COMMANDS,
        "failure_disclosure": {
            "path": "evals/failure-cases.md",
            "case_count": failure_case_count,
            "policy": "Keep representative failures visible and tied to regression checks.",
        },
        "limitations": [
            "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
            "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
            "Pending blind-review decisions are visible but do not count as human adjudication.",
            "World-class readiness remains false until external and human evidence gaps close.",
        ],
        "artifacts": {
            "json": "reports/benchmark_reproducibility.json",
            "markdown": "reports/benchmark_reproducibility.md",
        },
    }


def render_markdown(report: dict[str, Any]) -> str:
    summary = report["summary"]
    lines = [
        "# Benchmark Reproducibility",
        "",
        f"Generated at: `{report['generated_at']}`",
        f"Commit: `{report['commit']}`",
        f"Working tree dirty at generation: `{str(summary.get('working_tree_dirty')).lower()}`",
        f"Evidence bundle SHA256: `{summary.get('evidence_bundle_sha256', '')}`",
        "",
        "## Summary",
        "",
        f"- reproducibility ready: `{str(summary['reproducibility_ready']).lower()}`",
        f"- release lock ready: `{str(summary.get('release_lock_ready')).lower()}`",
        f"- methodology complete: `{str(summary['methodology_complete']).lower()}`",
        f"- required artifacts: `{summary['required_artifact_count']}`",
        f"- missing artifacts: `{summary['missing_artifact_count']}`",
        f"- source contract sha256: `{summary.get('source_contract_sha256', '')[:12]}`",
        f"- archive sha256: `{summary.get('archive_sha256', '')[:12]}`",
        f"- output cases: `{summary['output_case_count']}`",
        f"- disclosed failure cases: `{summary['failure_disclosure_count']}`",
        f"- reproduction commands: `{summary['command_count']}`",
        f"- provider evidence complete: `{str(summary['provider_evidence_complete']).lower()}`",
        f"- human review complete: `{str(summary['human_review_complete']).lower()}`",
        f"- world-class ready: `{str(summary['world_class_ready']).lower()}`",
        f"- public claim ready: `{str(summary['public_claim_ready']).lower()}`",
        f"- public claim blockers: `{summary['public_claim_blocker_count']}`",
        f"- changed files at generation: `{summary.get('changed_file_count')}`",
        "",
        "This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.",
        "",
        "## Public Claim Boundary",
        "",
        f"- ready: `{str(report.get('public_claim', {}).get('ready')).lower()}`",
        f"- scope: {report.get('public_claim', {}).get('scope', 'public benchmark claim')}",
        f"- policy: {report.get('public_claim', {}).get('policy', '')}",
        "",
        "| Blocker |",
        "| --- |",
    ]
    blockers = report.get("public_claim", {}).get("blockers", [])
    if blockers:
        lines.extend(f"| {item} |" for item in blockers)
    else:
        lines.append("| none |")
    lines.extend(
        [
            "",
            "## Release Lock",
            "",
            f"- ready: `{str(report.get('release_lock', {}).get('ready')).lower()}`",
            f"- reason: {report.get('release_lock', {}).get('reason', 'unknown')}",
            f"- status scope: {report.get('release_lock', {}).get('status_scope', 'generation-time status')}",
            "",
            "## Evidence Bundle",
        ]
    )
    lines.extend(
        [
            "",
            f"- algorithm: `{report.get('evidence_bundle', {}).get('algorithm', '')}`",
            f"- artifacts: `{report.get('evidence_bundle', {}).get('existing_count', 0)}` / `{report.get('evidence_bundle', {}).get('artifact_count', 0)}`",
            f"- sha256: `{report.get('evidence_bundle', {}).get('sha256', '')}`",
            "",
            "## Methodology Sections",
            "",
            "| Section | Status |",
            "| --- | --- |",
        ]
    )
    for section in report["methodology"]["sections"]:
        lines.append(f"| `{section['heading']}` | {'present' if section['exists'] else 'missing'} |")
    lines.extend(["", "## Required Artifacts", "", "| Label | Path | Status | SHA256 |", "| --- | --- | --- | --- |"])
    for artifact in report["artifacts_checked"]:
        digest = artifact.get("sha256", "")
        lines.append(
            f"| {artifact['label']} | `{artifact['path']}` | {'present' if artifact['exists'] else 'missing'} | `{digest[:12]}` |"
        )
    lines.extend(["", "## Reproduction Commands", ""])
    for command in report["reproduction_commands"]:
        lines.append(f"- `{command['command']}`")
        lines.append(f"  - evidence: `{command['evidence']}`")
    lines.extend(["", "## Failure Disclosure", ""])
    disclosure = report["failure_disclosure"]
    lines.append(f"- path: `{disclosure['path']}`")
    lines.append(f"- disclosed cases: `{disclosure['case_count']}`")
    lines.append(f"- policy: {disclosure['policy']}")
    lines.extend(["", "## Limits", ""])
    lines.extend(f"- {item}" for item in report["limitations"])
    return "\n".join(lines).rstrip() + "\n"


def main() -> None:
    parser = argparse.ArgumentParser(description="Render benchmark reproducibility evidence.")
    parser.add_argument("skill_dir", nargs="?", default=".")
    parser.add_argument("--output-json", default="reports/benchmark_reproducibility.json")
    parser.add_argument("--output-md", default="reports/benchmark_reproducibility.md")
    parser.add_argument("--generated-at", default=date.today().isoformat())
    args = parser.parse_args()

    skill_dir = Path(args.skill_dir).resolve()
    report = build_report(skill_dir, args.generated_at)
    output_json = Path(args.output_json)
    output_md = Path(args.output_md)
    if not output_json.is_absolute():
        output_json = skill_dir / output_json
    if not output_md.is_absolute():
        output_md = skill_dir / output_md
    output_json.parent.mkdir(parents=True, exist_ok=True)
    output_md.parent.mkdir(parents=True, exist_ok=True)
    output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
    output_md.write_text(render_markdown(report), encoding="utf-8")
    print(json.dumps(report, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()