#!/usr/bin/env python3 import json import shutil import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_benchmark_reproducibility.py" TMP = ROOT / "tests" / "tmp_benchmark_reproducibility" sys.path.insert(0, str(ROOT / "scripts")) from evidence_consistency_core import beta_public_claim_split_values # noqa: E402 from render_benchmark_reproducibility import beta_deferred_evidence # noqa: E402 def main() -> None: shutil.rmtree(TMP, ignore_errors=True) TMP.mkdir(parents=True, exist_ok=True) subprocess.run( [ sys.executable, str(ROOT / "scripts" / "render_world_class_evidence_ledger.py"), str(ROOT), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) subprocess.run( [ sys.executable, str(ROOT / "scripts" / "render_world_class_evidence_intake.py"), str(ROOT), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) subprocess.run( [ sys.executable, str(ROOT / "scripts" / "render_world_class_preflight.py"), str(ROOT), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) submission_review_proc = subprocess.run( [ sys.executable, str(ROOT / "scripts" / "render_world_class_submission_review.py"), str(ROOT), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=False, ) assert submission_review_proc.returncode in {0, 2}, submission_review_proc.stderr subprocess.run( [ sys.executable, str(ROOT / "scripts" / "render_world_class_operator_runbook.py"), str(ROOT), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) subprocess.run( [ sys.executable, str(ROOT / "scripts" / "render_world_class_claim_guard.py"), str(ROOT), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) subprocess.run( [ sys.executable, str(ROOT / "scripts" / "python_compat_check.py"), str(ROOT), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) output_json = TMP / "benchmark_reproducibility.json" output_md = TMP / "benchmark_reproducibility.md" proc = subprocess.run( [ sys.executable, str(SCRIPT), str(ROOT), "--output-json", str(output_json), "--output-md", str(output_md), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) payload = json.loads(proc.stdout) assert payload["schema_version"] == "1.0", payload assert payload["ok"] is True, payload assert payload["summary"]["reproducibility_ready"] is True, payload assert payload["summary"]["release_lock_ready"] == (payload["git_status"]["source_dirty"] is False), payload assert payload["summary"]["methodology_complete"] is True, payload assert payload["summary"]["missing_artifact_count"] == 0, payload assert len(payload["summary"]["evidence_bundle_sha256"]) == 64, payload assert payload["summary"]["evidence_bundle_sha256"] == payload["evidence_bundle"]["sha256"], payload assert payload["summary"]["source_contract_sha256"], payload assert payload["summary"]["archive_sha256"], payload assert payload["summary"]["output_case_count"] >= 5, payload assert payload["summary"]["failure_disclosure_count"] >= 1, payload assert payload["summary"]["command_count"] >= 10, payload assert "working_tree_dirty" in payload["summary"], payload assert "source_tree_dirty" in payload["summary"], payload assert "generated_tree_dirty" in payload["summary"], payload assert payload["summary"]["source_changed_file_count"] == payload["git_status"]["source_changed_file_count"], payload assert payload["summary"]["generated_changed_file_count"] == payload["git_status"][ "generated_changed_file_count" ], payload assert payload["git_status"]["available"] is True, payload assert payload["git_status"]["scope"] == "generation-time status before this report is written", payload assert isinstance(payload["git_status"]["generated_dirty_prefixes"], list), payload assert payload["release_lock"]["status_scope"] == "generation-time status before this report is written", payload assert payload["release_lock"]["commit"] == payload["commit"], payload assert payload["release_lock"]["source_changed_file_count"] == payload["git_status"][ "source_changed_file_count" ], payload assert payload["release_lock"]["generated_changed_file_count"] == payload["git_status"][ "generated_changed_file_count" ], payload assert payload["evidence_bundle"]["algorithm"] == "sha256(path,label,exists,artifact_sha256)", payload assert payload["evidence_bundle"]["artifact_count"] == payload["summary"]["required_artifact_count"], payload assert payload["evidence_bundle"]["existing_count"] == payload["summary"]["required_artifact_count"], payload assert payload["evidence_bundle"]["missing_count"] == 0, payload assert payload["evidence_bundle"]["missing_paths"] == [], payload assert payload["summary"]["provider_evidence_complete"] is True, payload assert payload["summary"]["human_review_complete"] is False, payload assert payload["summary"]["world_class_ready"] is False, payload assert payload["summary"]["world_class_source_check_count"] >= 13, payload assert payload["summary"]["world_class_source_pass_count"] >= 6, payload assert payload["summary"]["world_class_source_blocked_count"] >= 6, payload assert ( payload["summary"]["world_class_source_pass_count"] + payload["summary"]["world_class_source_blocked_count"] == payload["summary"]["world_class_source_check_count"] ), payload expected_beta_ready = ( payload["summary"]["reproducibility_ready"] and payload["summary"]["release_lock_ready"] and payload["summary"]["provider_evidence_complete"] ) assert payload["summary"]["beta_test_ready"] is expected_beta_ready, payload assert payload["summary"]["beta_test_blocker_count"] == len(payload["beta_test_release"]["blockers"]), payload assert payload["summary"]["beta_test_deferred_evidence_count"] == len( payload["beta_test_release"]["allowed_deferred_evidence"] ), payload assert payload["beta_test_release"]["ready"] is expected_beta_ready, payload["beta_test_release"] assert "beta/public test release" in payload["beta_test_release"]["scope"], payload["beta_test_release"] assert "Human blind-review" in payload["beta_test_release"]["policy"], payload["beta_test_release"] assert "do not claim world-class" in payload["beta_test_release"]["required_wording"], payload[ "beta_test_release" ] ledger_payload = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8")) pending_ledger_keys = { item["key"] for item in ledger_payload["entries"] if item.get("status") == "pending" } deferred_keys = {item["key"] for item in payload["beta_test_release"]["allowed_deferred_evidence"]} assert deferred_keys == pending_ledger_keys, payload["beta_test_release"] assert "human-adjudication" in deferred_keys, payload["beta_test_release"] assert "provider-holdout" in deferred_keys, payload["beta_test_release"] assert "provider-ledger-review" not in deferred_keys, payload["beta_test_release"] assert not any("human blind-review" in item for item in payload["beta_test_release"]["blockers"]), payload[ "beta_test_release" ] synthetic_deferred = beta_deferred_evidence( { "entries": [ {"key": "provider-holdout", "label": "Provider Holdout", "status": "accepted"}, {"key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "pending"}, ] } ) assert {item["key"] for item in synthetic_deferred} == {"native-client-telemetry"}, synthetic_deferred expected_beta_split, actual_beta_split = beta_public_claim_split_values( { "summary": { "reproducibility_ready": True, "release_lock_ready": True, "provider_evidence_complete": True, "beta_test_ready": True, "public_claim_ready": False, "human_review_complete": True, }, "beta_test_release": { "ready": True, "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", "allowed_deferred_evidence": [{"key": "native-client-telemetry", "reason": "still pending"}], }, }, { "summary": {"ready_to_claim_world_class": False}, "entries": [ {"key": "human-adjudication", "status": "accepted"}, {"key": "native-client-telemetry", "status": "pending"}, ], }, {"data": {"output_review_adjudication": {"summary": {"pair_count": 5, "pending_count": 0}}}}, ) assert expected_beta_split == actual_beta_split, (expected_beta_split, actual_beta_split) assert expected_beta_split["human_review_complete"] is True, expected_beta_split assert expected_beta_split["deferred_human_review"] is False, expected_beta_split assert payload["summary"]["public_claim_ready"] is False, payload minimum_blockers = 3 if payload["summary"]["release_lock_ready"] else 4 assert payload["summary"]["public_claim_blocker_count"] >= minimum_blockers, payload assert payload["public_claim"]["ready"] is False, payload["public_claim"] if payload["summary"]["release_lock_ready"]: assert not any("release lock" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] else: assert any("release lock" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] assert not any("provider-backed model holdout evidence is incomplete" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] assert any("human blind-review" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] assert any("world-class evidence" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] assert any("world-class source checks" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"] assert "complete source checks" in payload["public_claim"]["policy"], payload["public_claim"] headings = {item["heading"]: item["exists"] for item in payload["methodology"]["sections"]} assert headings["## Benchmark Types"], headings assert headings["## Failure Disclosure"], headings artifacts = {item["path"]: item for item in payload["artifacts_checked"]} assert artifacts["reports/benchmark_methodology.md"]["exists"], artifacts assert artifacts["evals/failure-cases.md"]["exists"], artifacts assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts assert artifacts["reports/world_class_evidence_ledger.json"]["exists"], artifacts assert artifacts["reports/world_class_evidence_intake.json"]["exists"], artifacts assert artifacts["reports/world_class_evidence_preflight.json"]["exists"], artifacts assert artifacts["reports/world_class_submission_review.json"]["exists"], artifacts assert artifacts["reports/world_class_operator_runbook.json"]["exists"], artifacts assert artifacts["reports/world_class_operator_runbook.md"]["exists"], artifacts assert artifacts["reports/world_class_operator_runbook.html"]["exists"], artifacts assert artifacts["reports/world_class_claim_guard.json"]["exists"], artifacts assert artifacts["reports/python_compatibility.json"]["exists"], artifacts assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py world-class-claim-guard ." for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py python-compat ." for command in payload["reproduction_commands"]), payload assert any(command["command"] == "python3 scripts/yao.py evidence-consistency ." for command in payload["reproduction_commands"]), payload assert any("Provider-backed model holdout source evidence is complete" in item for item in payload["limitations"]), payload["limitations"] markdown = output_md.read_text(encoding="utf-8") assert "Benchmark Reproducibility" in markdown, markdown assert "Evidence bundle SHA256" in markdown, markdown assert "release lock ready" in markdown, markdown assert "Source tree dirty at generation" in markdown, markdown assert "Generated evidence dirty at generation" in markdown, markdown assert "source changed files at generation" in markdown, markdown assert "generated changed files at generation" in markdown, markdown assert "world-class source checks" in markdown, markdown assert "beta test ready" in markdown, markdown assert "## Beta Test Boundary" in markdown, markdown assert "human-adjudication" in markdown, markdown assert "do not claim world-class" in markdown, markdown assert "public claim ready: `false`" in markdown, markdown assert "## Public Claim Boundary" in markdown, markdown assert "provider-backed model holdout evidence is incomplete" not in markdown, markdown assert "Provider-backed model holdout source evidence is complete" in markdown, markdown assert "world-class source checks are not all accepted" in markdown, markdown assert "## Release Lock" in markdown, markdown assert "## Evidence Bundle" in markdown, markdown assert "reports/benchmark_methodology.md" in markdown, markdown assert "reports/world_class_evidence_preflight.json" in markdown, markdown assert "reports/world_class_operator_runbook.html" in markdown, markdown assert "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" in markdown, markdown assert "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" in markdown, markdown assert "python3 scripts/yao.py evidence-consistency ." in markdown, markdown assert "make ci-test" in markdown, markdown print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()