diff --git a/scripts/evidence_consistency_core.py b/scripts/evidence_consistency_core.py index dd0620a..9a0dc61 100644 --- a/scripts/evidence_consistency_core.py +++ b/scripts/evidence_consistency_core.py @@ -217,6 +217,59 @@ def gate_by_key(review_studio: dict[str, Any], key: str) -> dict[str, Any]: return {} +def beta_public_claim_split_values( + benchmark: dict[str, Any], + ledger: dict[str, Any], + review_studio: dict[str, Any], +) -> tuple[dict[str, Any], dict[str, Any]]: + benchmark_summary = nested(benchmark, ["summary"], {}) + ledger_summary = nested(ledger, ["summary"], {}) + expected_beta_ready = ( + bool(benchmark_summary.get("reproducibility_ready")) + and bool(benchmark_summary.get("release_lock_ready")) + and bool(benchmark_summary.get("provider_evidence_complete")) + ) + output_review_summary = nested(review_studio, ["data", "output_review_adjudication", "summary"], {}) + review_pair_count = as_int(output_review_summary.get("pair_count")) if isinstance(output_review_summary, dict) else None + review_pending_count = as_int(output_review_summary.get("pending_count")) if isinstance(output_review_summary, dict) else None + expected_human_review_complete = ( + review_pair_count is not None and review_pair_count > 0 and review_pending_count == 0 + ) + ledger_entries = ledger.get("entries", []) if isinstance(ledger, dict) else [] + pending_ledger_keys = sorted( + str(entry.get("key", "")) + for entry in ledger_entries + if isinstance(entry, dict) and entry.get("status") == "pending" and str(entry.get("key", "")) + ) + beta_boundary = { + "beta_test_ready": expected_beta_ready, + "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"), + "human_review_complete": expected_human_review_complete, + "beta_release_ready": expected_beta_ready, + "beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "deferred_evidence_keys": pending_ledger_keys, + "deferred_human_review": "human-adjudication" in pending_ledger_keys, + } + beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {} + deferred_keys = sorted( + str(item.get("key", "")) + for item in beta_release.get("allowed_deferred_evidence", []) + if isinstance(item, dict) and str(item.get("key", "")) + ) + actual_beta_boundary = { + "beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None, + "public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None, + "human_review_complete": benchmark_summary.get("human_review_complete") + if isinstance(benchmark_summary, dict) + else None, + "beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None, + "beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None, + "deferred_evidence_keys": deferred_keys, + "deferred_human_review": "human-adjudication" in deferred_keys, + } + return beta_boundary, actual_beta_boundary + + def report_contract(payload: dict[str, Any]) -> dict[str, Any]: contract = payload.get("report_contract") return contract if isinstance(contract, dict) else {} diff --git a/scripts/render_benchmark_reproducibility.py b/scripts/render_benchmark_reproducibility.py index 4e486e7..d422c6d 100644 --- a/scripts/render_benchmark_reproducibility.py +++ b/scripts/render_benchmark_reproducibility.py @@ -326,36 +326,48 @@ def beta_test_blockers( return blockers -def beta_deferred_evidence(human_review_complete: bool, world_class_ready: bool) -> list[dict[str, str]]: +BETA_DEFERRED_EVIDENCE = { + "provider-holdout": { + "label": "Provider holdout ledger review", + "reason": "Provider-backed source evidence exists, but formal ledger submission and reviewer acceptance are still pending before public claims.", + }, + "human-adjudication": { + "label": "Human blind-review adjudication", + "reason": "Human adjudication evidence is still pending; deferred for beta/public testing and still required before superiority, fully-reviewed, or world-class claims.", + }, + "native-permission-enforcement": { + "label": "Native permission enforcement evidence", + "reason": "Native enforcement proof is still pending; deferred for beta/public testing and still required before world-class claims.", + }, + "native-client-telemetry": { + "label": "Real client telemetry evidence", + "reason": "Real client telemetry is still pending; deferred for beta/public testing and still required before world-class claims.", + }, +} + + +def beta_deferred_evidence(world_class_ledger: dict[str, Any]) -> list[dict[str, str]]: deferred = [] - if not human_review_complete: + entries = world_class_ledger.get("entries", []) + if not isinstance(entries, list): + return deferred + for entry in entries: + if not isinstance(entry, dict) or entry.get("status") != "pending": + continue + key = str(entry.get("key") or "") + if not key: + continue + fallback = BETA_DEFERRED_EVIDENCE.get(key, {}) deferred.append( { - "key": "human-adjudication", - "label": "Human blind-review adjudication", - "reason": "Deferred for beta/public testing; still required before superiority, fully-reviewed, or world-class claims.", + "key": key, + "label": str(entry.get("label") or fallback.get("label") or key), + "reason": str( + fallback.get("reason") + or "Formal evidence is still pending; deferred for beta/public testing and still required before public claims." + ), } ) - if not world_class_ready: - deferred.extend( - [ - { - "key": "provider-ledger-review", - "label": "Independent provider ledger review", - "reason": "Provider source evidence may be complete, but independent ledger acceptance remains formal-claim evidence.", - }, - { - "key": "native-permission-enforcement", - "label": "Native permission enforcement evidence", - "reason": "Native enforcement proof is deferred for beta/public testing and remains required for world-class claims.", - }, - { - "key": "native-client-telemetry", - "label": "Real client telemetry evidence", - "reason": "Real client telemetry is deferred for beta/public testing and remains required for world-class claims.", - }, - ] - ) return deferred @@ -417,7 +429,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: release_lock["ready"], provider_evidence_complete, ) - beta_deferred = beta_deferred_evidence(human_review_complete, world_class_ready) + beta_deferred = beta_deferred_evidence(world_class_ledger) beta_test_ready = not beta_blockers limitations = [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index 19500c5..068c713 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -16,6 +16,7 @@ from evidence_consistency_core import ( REQUIRED_TEXT_REPORTS, add_check, as_int, + beta_public_claim_split_values, compare_summary_keys, compare_values, gate_by_key, @@ -450,35 +451,11 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]], detail="Benchmark reproducibility must not overstate public claim readiness.", ) - expected_beta_ready = ( - bool(benchmark_summary.get("reproducibility_ready")) - and bool(benchmark_summary.get("release_lock_ready")) - and bool(benchmark_summary.get("provider_evidence_complete")) + beta_boundary, actual_beta_boundary = beta_public_claim_split_values( + benchmark, + ledger, + review_studio, ) - beta_boundary = { - "beta_test_ready": expected_beta_ready, - "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"), - "human_review_complete": False, - "beta_release_ready": expected_beta_ready, - "beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", - "deferred_human_review": True, - } - beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {} - deferred_keys = { - str(item.get("key", "")) - for item in beta_release.get("allowed_deferred_evidence", []) - if isinstance(item, dict) - } - actual_beta_boundary = { - "beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None, - "public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None, - "human_review_complete": benchmark_summary.get("human_review_complete") - if isinstance(benchmark_summary, dict) - else None, - "beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None, - "beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None, - "deferred_human_review": "human-adjudication" in deferred_keys, - } compare_values( checks, key="benchmark-beta-public-claim-split", diff --git a/tests/review_studio_helpers.py b/tests/review_studio_helpers.py index aec2628..be697f5 100644 --- a/tests/review_studio_helpers.py +++ b/tests/review_studio_helpers.py @@ -170,6 +170,7 @@ def prepare_review_studio_inputs(tmp_root: Path) -> None: "render_world_class_operator_runbook.py", "render_world_class_claim_guard.py", "render_skill_os2_coverage.py", + "render_benchmark_reproducibility.py", ]: allowed_returncodes = (0, 2) if script_name == "render_world_class_submission_review.py" else (0,) run_script(script_name, ROOT, "--generated-at", "2026-06-13", allowed_returncodes=allowed_returncodes) diff --git a/tests/verify_benchmark_reproducibility.py b/tests/verify_benchmark_reproducibility.py index 1445fad..ce97343 100644 --- a/tests/verify_benchmark_reproducibility.py +++ b/tests/verify_benchmark_reproducibility.py @@ -9,6 +9,9 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_benchmark_reproducibility.py" TMP = ROOT / "tests" / "tmp_benchmark_reproducibility" +sys.path.insert(0, str(ROOT / "scripts")) +from evidence_consistency_core import beta_public_claim_split_values # noqa: E402 +from render_benchmark_reproducibility import beta_deferred_evidence # noqa: E402 def main() -> None: @@ -189,11 +192,55 @@ def main() -> None: assert "do not claim world-class" in payload["beta_test_release"]["required_wording"], payload[ "beta_test_release" ] + ledger_payload = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8")) + pending_ledger_keys = { + item["key"] for item in ledger_payload["entries"] if item.get("status") == "pending" + } deferred_keys = {item["key"] for item in payload["beta_test_release"]["allowed_deferred_evidence"]} + assert deferred_keys == pending_ledger_keys, payload["beta_test_release"] assert "human-adjudication" in deferred_keys, payload["beta_test_release"] + assert "provider-holdout" in deferred_keys, payload["beta_test_release"] + assert "provider-ledger-review" not in deferred_keys, payload["beta_test_release"] assert not any("human blind-review" in item for item in payload["beta_test_release"]["blockers"]), payload[ "beta_test_release" ] + synthetic_deferred = beta_deferred_evidence( + { + "entries": [ + {"key": "provider-holdout", "label": "Provider Holdout", "status": "accepted"}, + {"key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "pending"}, + ] + } + ) + assert {item["key"] for item in synthetic_deferred} == {"native-client-telemetry"}, synthetic_deferred + expected_beta_split, actual_beta_split = beta_public_claim_split_values( + { + "summary": { + "reproducibility_ready": True, + "release_lock_ready": True, + "provider_evidence_complete": True, + "beta_test_ready": True, + "public_claim_ready": False, + "human_review_complete": True, + }, + "beta_test_release": { + "ready": True, + "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "allowed_deferred_evidence": [{"key": "native-client-telemetry", "reason": "still pending"}], + }, + }, + { + "summary": {"ready_to_claim_world_class": False}, + "entries": [ + {"key": "human-adjudication", "status": "accepted"}, + {"key": "native-client-telemetry", "status": "pending"}, + ], + }, + {"data": {"output_review_adjudication": {"summary": {"pair_count": 5, "pending_count": 0}}}}, + ) + assert expected_beta_split == actual_beta_split, (expected_beta_split, actual_beta_split) + assert expected_beta_split["human_review_complete"] is True, expected_beta_split + assert expected_beta_split["deferred_human_review"] is False, expected_beta_split assert payload["summary"]["public_claim_ready"] is False, payload minimum_blockers = 3 if payload["summary"]["release_lock_ready"] else 4 assert payload["summary"]["public_claim_blocker_count"] >= minimum_blockers, payload diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index 73869ea..d420960 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -202,6 +202,7 @@ def main() -> None: beta_split = checks["benchmark-beta-public-claim-split"]["actual"] assert beta_split["public_claim_ready"] is False, beta_split assert beta_split["human_review_complete"] is False, beta_split + assert beta_split["deferred_evidence_keys"] == ["human-adjudication", "native-client-telemetry", "native-permission-enforcement", "provider-holdout"], beta_split assert beta_split["deferred_human_review"] is True, beta_split assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"] assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[ diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 4bf2cae..1cacb83 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -222,6 +222,13 @@ def main() -> None: beta_release = full_payload["data"]["benchmark_reproducibility"]["beta_test_release"] assert beta_release["ready"] is expected_beta_ready, beta_release assert not any("human blind-review" in item for item in beta_release["blockers"]), beta_release + pending_ledger_keys = { + item["key"] + for item in full_payload["data"]["world_class_evidence_ledger"]["entries"] + if item.get("status") == "pending" + } + deferred_keys = {item["key"] for item in beta_release["allowed_deferred_evidence"]} + assert deferred_keys == pending_ledger_keys, beta_release assert any(item["key"] == "human-adjudication" for item in beta_release["allowed_deferred_evidence"]), ( beta_release )