#!/usr/bin/env python3
import json
import shutil
import subprocess
import sys
from pathlib import Path


ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "render_benchmark_reproducibility.py"
TMP = ROOT / "tests" / "tmp_benchmark_reproducibility"
sys.path.insert(0, str(ROOT / "scripts"))
from evidence_consistency_core import beta_public_claim_split_values  # noqa: E402
from render_benchmark_reproducibility import beta_deferred_evidence  # noqa: E402


def main() -> None:
    shutil.rmtree(TMP, ignore_errors=True)
    TMP.mkdir(parents=True, exist_ok=True)
    subprocess.run(
        [
            sys.executable,
            str(ROOT / "scripts" / "render_world_class_evidence_ledger.py"),
            str(ROOT),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    )
    subprocess.run(
        [
            sys.executable,
            str(ROOT / "scripts" / "render_world_class_evidence_intake.py"),
            str(ROOT),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    )
    subprocess.run(
        [
            sys.executable,
            str(ROOT / "scripts" / "render_world_class_preflight.py"),
            str(ROOT),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    )
    submission_review_proc = subprocess.run(
        [
            sys.executable,
            str(ROOT / "scripts" / "render_world_class_submission_review.py"),
            str(ROOT),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=False,
    )
    assert submission_review_proc.returncode in {0, 2}, submission_review_proc.stderr
    subprocess.run(
        [
            sys.executable,
            str(ROOT / "scripts" / "render_world_class_operator_runbook.py"),
            str(ROOT),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    )
    subprocess.run(
        [
            sys.executable,
            str(ROOT / "scripts" / "render_world_class_claim_guard.py"),
            str(ROOT),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    )
    subprocess.run(
        [
            sys.executable,
            str(ROOT / "scripts" / "python_compat_check.py"),
            str(ROOT),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    )
    output_json = TMP / "benchmark_reproducibility.json"
    output_md = TMP / "benchmark_reproducibility.md"
    proc = subprocess.run(
        [
            sys.executable,
            str(SCRIPT),
            str(ROOT),
            "--output-json",
            str(output_json),
            "--output-md",
            str(output_md),
            "--generated-at",
            "2026-06-13",
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        check=True,
    )
    payload = json.loads(proc.stdout)
    assert payload["schema_version"] == "1.0", payload
    assert payload["ok"] is True, payload
    assert payload["summary"]["reproducibility_ready"] is True, payload
    assert payload["summary"]["release_lock_ready"] == (payload["git_status"]["source_dirty"] is False), payload
    assert payload["summary"]["methodology_complete"] is True, payload
    assert payload["summary"]["missing_artifact_count"] == 0, payload
    assert len(payload["summary"]["evidence_bundle_sha256"]) == 64, payload
    assert payload["summary"]["evidence_bundle_sha256"] == payload["evidence_bundle"]["sha256"], payload
    assert payload["summary"]["source_contract_sha256"], payload
    assert payload["summary"]["archive_sha256"], payload
    assert payload["summary"]["output_case_count"] >= 5, payload
    assert payload["summary"]["failure_disclosure_count"] >= 1, payload
    assert payload["summary"]["command_count"] >= 10, payload
    assert "working_tree_dirty" in payload["summary"], payload
    assert "source_tree_dirty" in payload["summary"], payload
    assert "generated_tree_dirty" in payload["summary"], payload
    assert payload["summary"]["source_changed_file_count"] == payload["git_status"]["source_changed_file_count"], payload
    assert payload["summary"]["generated_changed_file_count"] == payload["git_status"][
        "generated_changed_file_count"
    ], payload
    assert payload["git_status"]["available"] is True, payload
    assert payload["git_status"]["scope"] == "generation-time status before this report is written", payload
    assert isinstance(payload["git_status"]["generated_dirty_prefixes"], list), payload
    assert payload["release_lock"]["status_scope"] == "generation-time status before this report is written", payload
    assert payload["release_lock"]["commit"] == payload["commit"], payload
    assert payload["release_lock"]["source_changed_file_count"] == payload["git_status"][
        "source_changed_file_count"
    ], payload
    assert payload["release_lock"]["generated_changed_file_count"] == payload["git_status"][
        "generated_changed_file_count"
    ], payload
    assert payload["evidence_bundle"]["algorithm"] == "sha256(path,label,exists,artifact_sha256)", payload
    assert payload["evidence_bundle"]["artifact_count"] == payload["summary"]["required_artifact_count"], payload
    assert payload["evidence_bundle"]["existing_count"] == payload["summary"]["required_artifact_count"], payload
    assert payload["evidence_bundle"]["missing_count"] == 0, payload
    assert payload["evidence_bundle"]["missing_paths"] == [], payload
    assert payload["summary"]["provider_evidence_complete"] is True, payload
    assert payload["summary"]["human_review_complete"] is False, payload
    assert payload["summary"]["world_class_ready"] is False, payload
    assert payload["summary"]["world_class_source_check_count"] >= 13, payload
    assert payload["summary"]["world_class_source_pass_count"] >= 6, payload
    assert payload["summary"]["world_class_source_blocked_count"] >= 6, payload
    assert (
        payload["summary"]["world_class_source_pass_count"]
        + payload["summary"]["world_class_source_blocked_count"]
        == payload["summary"]["world_class_source_check_count"]
    ), payload
    expected_beta_ready = (
        payload["summary"]["reproducibility_ready"]
        and payload["summary"]["release_lock_ready"]
        and payload["summary"]["provider_evidence_complete"]
    )
    assert payload["summary"]["beta_test_ready"] is expected_beta_ready, payload
    assert payload["summary"]["beta_test_blocker_count"] == len(payload["beta_test_release"]["blockers"]), payload
    assert payload["summary"]["beta_test_deferred_evidence_count"] == len(
        payload["beta_test_release"]["allowed_deferred_evidence"]
    ), payload
    assert payload["beta_test_release"]["ready"] is expected_beta_ready, payload["beta_test_release"]
    assert "beta/public test release" in payload["beta_test_release"]["scope"], payload["beta_test_release"]
    assert "Human blind-review" in payload["beta_test_release"]["policy"], payload["beta_test_release"]
    assert "do not claim world-class" in payload["beta_test_release"]["required_wording"], payload[
        "beta_test_release"
    ]
    ledger_payload = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8"))
    pending_ledger_keys = {
        item["key"] for item in ledger_payload["entries"] if item.get("status") == "pending"
    }
    deferred_keys = {item["key"] for item in payload["beta_test_release"]["allowed_deferred_evidence"]}
    assert deferred_keys == pending_ledger_keys, payload["beta_test_release"]
    assert "human-adjudication" in deferred_keys, payload["beta_test_release"]
    assert "provider-holdout" in deferred_keys, payload["beta_test_release"]
    assert "provider-ledger-review" not in deferred_keys, payload["beta_test_release"]
    assert not any("human blind-review" in item for item in payload["beta_test_release"]["blockers"]), payload[
        "beta_test_release"
    ]
    synthetic_deferred = beta_deferred_evidence(
        {
            "entries": [
                {"key": "provider-holdout", "label": "Provider Holdout", "status": "accepted"},
                {"key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "pending"},
            ]
        }
    )
    assert {item["key"] for item in synthetic_deferred} == {"native-client-telemetry"}, synthetic_deferred
    expected_beta_split, actual_beta_split = beta_public_claim_split_values(
        {
            "summary": {
                "reproducibility_ready": True,
                "release_lock_ready": True,
                "provider_evidence_complete": True,
                "beta_test_ready": True,
                "public_claim_ready": False,
                "human_review_complete": True,
            },
            "beta_test_release": {
                "ready": True,
                "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
                "allowed_deferred_evidence": [{"key": "native-client-telemetry", "reason": "still pending"}],
            },
        },
        {
            "summary": {"ready_to_claim_world_class": False},
            "entries": [
                {"key": "human-adjudication", "status": "accepted"},
                {"key": "native-client-telemetry", "status": "pending"},
            ],
        },
        {"data": {"output_review_adjudication": {"summary": {"pair_count": 5, "pending_count": 0}}}},
    )
    assert expected_beta_split == actual_beta_split, (expected_beta_split, actual_beta_split)
    assert expected_beta_split["human_review_complete"] is True, expected_beta_split
    assert expected_beta_split["deferred_human_review"] is False, expected_beta_split
    assert payload["summary"]["public_claim_ready"] is False, payload
    minimum_blockers = 3 if payload["summary"]["release_lock_ready"] else 4
    assert payload["summary"]["public_claim_blocker_count"] >= minimum_blockers, payload
    assert payload["public_claim"]["ready"] is False, payload["public_claim"]
    if payload["summary"]["release_lock_ready"]:
        assert not any("release lock" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"]
    else:
        assert any("release lock" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"]
    assert not any("provider-backed model holdout evidence is incomplete" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"]
    assert any("human blind-review" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"]
    assert any("world-class evidence" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"]
    assert any("world-class source checks" in item for item in payload["public_claim"]["blockers"]), payload["public_claim"]
    assert "complete source checks" in payload["public_claim"]["policy"], payload["public_claim"]
    headings = {item["heading"]: item["exists"] for item in payload["methodology"]["sections"]}
    assert headings["## Benchmark Types"], headings
    assert headings["## Failure Disclosure"], headings
    artifacts = {item["path"]: item for item in payload["artifacts_checked"]}
    assert artifacts["reports/benchmark_methodology.md"]["exists"], artifacts
    assert artifacts["evals/failure-cases.md"]["exists"], artifacts
    assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts
    assert artifacts["reports/world_class_evidence_ledger.json"]["exists"], artifacts
    assert artifacts["reports/world_class_evidence_intake.json"]["exists"], artifacts
    assert artifacts["reports/world_class_evidence_preflight.json"]["exists"], artifacts
    assert artifacts["reports/world_class_submission_review.json"]["exists"], artifacts
    assert artifacts["reports/world_class_operator_runbook.json"]["exists"], artifacts
    assert artifacts["reports/world_class_operator_runbook.md"]["exists"], artifacts
    assert artifacts["reports/world_class_operator_runbook.html"]["exists"], artifacts
    assert artifacts["reports/world_class_claim_guard.json"]["exists"], artifacts
    assert artifacts["reports/python_compatibility.json"]["exists"], artifacts
    assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py world-class-claim-guard ." for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py python-compat ." for command in payload["reproduction_commands"]), payload
    assert any(command["command"] == "python3 scripts/yao.py evidence-consistency ." for command in payload["reproduction_commands"]), payload
    assert any("Provider-backed model holdout source evidence is complete" in item for item in payload["limitations"]), payload["limitations"]
    markdown = output_md.read_text(encoding="utf-8")
    assert "Benchmark Reproducibility" in markdown, markdown
    assert "Evidence bundle SHA256" in markdown, markdown
    assert "release lock ready" in markdown, markdown
    assert "Source tree dirty at generation" in markdown, markdown
    assert "Generated evidence dirty at generation" in markdown, markdown
    assert "source changed files at generation" in markdown, markdown
    assert "generated changed files at generation" in markdown, markdown
    assert "world-class source checks" in markdown, markdown
    assert "beta test ready" in markdown, markdown
    assert "## Beta Test Boundary" in markdown, markdown
    assert "human-adjudication" in markdown, markdown
    assert "do not claim world-class" in markdown, markdown
    assert "public claim ready: `false`" in markdown, markdown
    assert "## Public Claim Boundary" in markdown, markdown
    assert "provider-backed model holdout evidence is incomplete" not in markdown, markdown
    assert "Provider-backed model holdout source evidence is complete" in markdown, markdown
    assert "world-class source checks are not all accepted" in markdown, markdown
    assert "## Release Lock" in markdown, markdown
    assert "## Evidence Bundle" in markdown, markdown
    assert "reports/benchmark_methodology.md" in markdown, markdown
    assert "reports/world_class_evidence_preflight.json" in markdown, markdown
    assert "reports/world_class_operator_runbook.html" in markdown, markdown
    assert "python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions" in markdown, markdown
    assert "python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions" in markdown, markdown
    assert "python3 scripts/yao.py evidence-consistency ." in markdown, markdown
    assert "make ci-test" in markdown, markdown
    print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()
