diff --git a/scripts/evidence_consistency_core.py b/scripts/evidence_consistency_core.py index ecb6e4d2..dd0620a2 100644 --- a/scripts/evidence_consistency_core.py +++ b/scripts/evidence_consistency_core.py @@ -41,6 +41,9 @@ BENCHMARK_SUMMARY_KEYS = [ "world_class_source_check_count", "world_class_source_pass_count", "world_class_source_blocked_count", + "beta_test_ready", + "beta_test_blocker_count", + "beta_test_deferred_evidence_count", "public_claim_ready", "public_claim_blocker_count", ] diff --git a/scripts/render_benchmark_reproducibility.py b/scripts/render_benchmark_reproducibility.py index 416fc848..4e486e78 100644 --- a/scripts/render_benchmark_reproducibility.py +++ b/scripts/render_benchmark_reproducibility.py @@ -311,6 +311,54 @@ def public_claim_blockers( return blockers +def beta_test_blockers( + local_reproducibility_ready: bool, + release_lock_ready: bool, + provider_evidence_complete: bool, +) -> list[str]: + blockers = [] + if not local_reproducibility_ready: + blockers.append("local benchmark reproducibility is incomplete") + if not release_lock_ready: + blockers.append("release lock is not clean or commit is unavailable") + if not provider_evidence_complete: + blockers.append("provider-backed model holdout source evidence is incomplete") + return blockers + + +def beta_deferred_evidence(human_review_complete: bool, world_class_ready: bool) -> list[dict[str, str]]: + deferred = [] + if not human_review_complete: + deferred.append( + { + "key": "human-adjudication", + "label": "Human blind-review adjudication", + "reason": "Deferred for beta/public testing; still required before superiority, fully-reviewed, or world-class claims.", + } + ) + if not world_class_ready: + deferred.extend( + [ + { + "key": "provider-ledger-review", + "label": "Independent provider ledger review", + "reason": "Provider source evidence may be complete, but independent ledger acceptance remains formal-claim evidence.", + }, + { + "key": "native-permission-enforcement", + "label": "Native permission enforcement evidence", + "reason": "Native enforcement proof is deferred for beta/public testing and remains required for world-class claims.", + }, + { + "key": "native-client-telemetry", + "label": "Real client telemetry evidence", + "reason": "Real client telemetry is deferred for beta/public testing and remains required for world-class claims.", + }, + ] + ) + return deferred + + def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: reports = skill_dir / "reports" output_quality = load_json(reports / "output_quality_scorecard.json") @@ -364,10 +412,18 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: world_class_source_blocked_count, ) public_claim_ready = not claim_blockers + beta_blockers = beta_test_blockers( + local_reproducibility_ready, + release_lock["ready"], + provider_evidence_complete, + ) + beta_deferred = beta_deferred_evidence(human_review_complete, world_class_ready) + beta_test_ready = not beta_blockers limitations = [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", "Pending blind-review decisions are visible but do not count as human adjudication.", "World-class readiness remains false until external and human evidence gaps close.", + "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims.", ] if provider_evidence_complete: limitations.insert( @@ -411,6 +467,9 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: "world_class_source_check_count": world_class_source_check_count, "world_class_source_pass_count": world_class_source_pass_count, "world_class_source_blocked_count": world_class_source_blocked_count, + "beta_test_ready": beta_test_ready, + "beta_test_blocker_count": len(beta_blockers), + "beta_test_deferred_evidence_count": len(beta_deferred), "public_claim_ready": public_claim_ready, "public_claim_blocker_count": len(claim_blockers), "working_tree_dirty": status.get("dirty"), @@ -420,6 +479,14 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: "generated_tree_dirty": status.get("generated_dirty"), "generated_changed_file_count": status.get("generated_changed_file_count"), }, + "beta_test_release": { + "ready": beta_test_ready, + "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "blockers": beta_blockers, + "allowed_deferred_evidence": beta_deferred, + "policy": "Human blind-review, native permission enforcement, real client telemetry, and ledger acceptance may be deferred for beta/public testing, but public claims must remain blocked until those evidence entries are accepted.", + "required_wording": "Use beta, public test, or technical preview wording; do not claim world-class readiness, fully reviewed quality, or proven superiority over baseline.", + }, "public_claim": { "ready": public_claim_ready, "scope": "public benchmark or world-class readiness claim", @@ -473,6 +540,9 @@ def render_markdown(report: dict[str, Any]) -> str: f"- human review complete: `{str(summary['human_review_complete']).lower()}`", f"- world-class ready: `{str(summary['world_class_ready']).lower()}`", f"- world-class source checks: `{summary.get('world_class_source_pass_count', 0)}` pass / `{summary.get('world_class_source_check_count', 0)}` total; `{summary.get('world_class_source_blocked_count', 0)}` blocked", + f"- beta test ready: `{str(summary['beta_test_ready']).lower()}`", + f"- beta test blockers: `{summary['beta_test_blocker_count']}`", + f"- beta deferred evidence: `{summary['beta_test_deferred_evidence_count']}`", f"- public claim ready: `{str(summary['public_claim_ready']).lower()}`", f"- public claim blockers: `{summary['public_claim_blocker_count']}`", f"- changed files at generation: `{summary.get('changed_file_count')}`", @@ -481,15 +551,49 @@ def render_markdown(report: dict[str, Any]) -> str: "", "This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.", "", - "## Public Claim Boundary", - "", - f"- ready: `{str(report.get('public_claim', {}).get('ready')).lower()}`", - f"- scope: {report.get('public_claim', {}).get('scope', 'public benchmark claim')}", - f"- policy: {report.get('public_claim', {}).get('policy', '')}", - "", - "| Blocker |", - "| --- |", ] + beta_release = report.get("beta_test_release", {}) + lines.extend( + [ + "## Beta Test Boundary", + "", + f"- ready: `{str(beta_release.get('ready')).lower()}`", + f"- scope: {beta_release.get('scope', 'beta/public test release')}", + f"- policy: {beta_release.get('policy', '')}", + f"- required wording: {beta_release.get('required_wording', '')}", + "", + "| Blocker |", + "| --- |", + ] + ) + beta_blockers = beta_release.get("blockers", []) + if beta_blockers: + lines.extend(f"| {item} |" for item in beta_blockers) + else: + lines.append("| none |") + lines.extend(["", "| Deferred evidence | Reason |", "| --- | --- |"]) + deferred = beta_release.get("allowed_deferred_evidence", []) + if deferred: + lines.extend(f"| `{item.get('key', '')}` | {item.get('reason', '')} |" for item in deferred) + else: + lines.append("| none | none |") + lines.extend( + [ + "", + "## Public Claim Boundary", + "", + ] + ) + lines.extend( + [ + f"- ready: `{str(report.get('public_claim', {}).get('ready')).lower()}`", + f"- scope: {report.get('public_claim', {}).get('scope', 'public benchmark claim')}", + f"- policy: {report.get('public_claim', {}).get('policy', '')}", + "", + "| Blocker |", + "| --- |", + ] + ) blockers = report.get("public_claim", {}).get("blockers", []) if blockers: lines.extend(f"| {item} |" for item in blockers) diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index 9a0e5b0f..19500c58 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -450,6 +450,46 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]], detail="Benchmark reproducibility must not overstate public claim readiness.", ) + expected_beta_ready = ( + bool(benchmark_summary.get("reproducibility_ready")) + and bool(benchmark_summary.get("release_lock_ready")) + and bool(benchmark_summary.get("provider_evidence_complete")) + ) + beta_boundary = { + "beta_test_ready": expected_beta_ready, + "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"), + "human_review_complete": False, + "beta_release_ready": expected_beta_ready, + "beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims", + "deferred_human_review": True, + } + beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {} + deferred_keys = { + str(item.get("key", "")) + for item in beta_release.get("allowed_deferred_evidence", []) + if isinstance(item, dict) + } + actual_beta_boundary = { + "beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None, + "public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None, + "human_review_complete": benchmark_summary.get("human_review_complete") + if isinstance(benchmark_summary, dict) + else None, + "beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None, + "beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None, + "deferred_human_review": "human-adjudication" in deferred_keys, + } + compare_values( + checks, + key="benchmark-beta-public-claim-split", + label="Benchmark separates beta testing from public claims", + expected=beta_boundary, + actual=actual_beta_boundary, + paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS["world_class_ledger"]], + detail=( + "Beta/public testing may proceed with human blind review deferred, but public world-class or superiority claims must remain blocked." + ), + ) preflight_boundary = { "pending_count": ledger_summary.get("pending_count"), "source_check_count": ledger_summary.get("source_check_count"), diff --git a/tests/verify_benchmark_reproducibility.py b/tests/verify_benchmark_reproducibility.py index 511842ef..1445fadd 100644 --- a/tests/verify_benchmark_reproducibility.py +++ b/tests/verify_benchmark_reproducibility.py @@ -173,6 +173,27 @@ def main() -> None: + payload["summary"]["world_class_source_blocked_count"] == payload["summary"]["world_class_source_check_count"] ), payload + expected_beta_ready = ( + payload["summary"]["reproducibility_ready"] + and payload["summary"]["release_lock_ready"] + and payload["summary"]["provider_evidence_complete"] + ) + assert payload["summary"]["beta_test_ready"] is expected_beta_ready, payload + assert payload["summary"]["beta_test_blocker_count"] == len(payload["beta_test_release"]["blockers"]), payload + assert payload["summary"]["beta_test_deferred_evidence_count"] == len( + payload["beta_test_release"]["allowed_deferred_evidence"] + ), payload + assert payload["beta_test_release"]["ready"] is expected_beta_ready, payload["beta_test_release"] + assert "beta/public test release" in payload["beta_test_release"]["scope"], payload["beta_test_release"] + assert "Human blind-review" in payload["beta_test_release"]["policy"], payload["beta_test_release"] + assert "do not claim world-class" in payload["beta_test_release"]["required_wording"], payload[ + "beta_test_release" + ] + deferred_keys = {item["key"] for item in payload["beta_test_release"]["allowed_deferred_evidence"]} + assert "human-adjudication" in deferred_keys, payload["beta_test_release"] + assert not any("human blind-review" in item for item in payload["beta_test_release"]["blockers"]), payload[ + "beta_test_release" + ] assert payload["summary"]["public_claim_ready"] is False, payload minimum_blockers = 3 if payload["summary"]["release_lock_ready"] else 4 assert payload["summary"]["public_claim_blocker_count"] >= minimum_blockers, payload @@ -221,6 +242,10 @@ def main() -> None: assert "source changed files at generation" in markdown, markdown assert "generated changed files at generation" in markdown, markdown assert "world-class source checks" in markdown, markdown + assert "beta test ready" in markdown, markdown + assert "## Beta Test Boundary" in markdown, markdown + assert "human-adjudication" in markdown, markdown + assert "do not claim world-class" in markdown, markdown assert "public claim ready: `false`" in markdown, markdown assert "## Public Claim Boundary" in markdown, markdown assert "provider-backed model holdout evidence is incomplete" not in markdown, markdown diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index 666cdce5..73869ea0 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -196,6 +196,13 @@ def main() -> None: assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"] assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"] assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"] + assert checks["benchmark-beta-public-claim-split"]["status"] == "pass", checks[ + "benchmark-beta-public-claim-split" + ] + beta_split = checks["benchmark-beta-public-claim-split"]["actual"] + assert beta_split["public_claim_ready"] is False, beta_split + assert beta_split["human_review_complete"] is False, beta_split + assert beta_split["deferred_human_review"] is True, beta_split assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"] assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[ "preflight-submission-kit-handoff" diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 9c4c6c16..4bf2caed 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -212,6 +212,19 @@ def main() -> None: assert benchmark_summary["reproducibility_ready"] is True, benchmark_summary assert benchmark_summary["release_lock_ready"] == (benchmark_summary["source_tree_dirty"] is False), benchmark_summary assert "generated_tree_dirty" in benchmark_summary, benchmark_summary + expected_beta_ready = ( + benchmark_summary["reproducibility_ready"] + and benchmark_summary["release_lock_ready"] + and benchmark_summary["provider_evidence_complete"] + ) + assert benchmark_summary["beta_test_ready"] is expected_beta_ready, benchmark_summary + assert benchmark_summary["beta_test_deferred_evidence_count"] >= 1, benchmark_summary + beta_release = full_payload["data"]["benchmark_reproducibility"]["beta_test_release"] + assert beta_release["ready"] is expected_beta_ready, beta_release + assert not any("human blind-review" in item for item in beta_release["blockers"]), beta_release + assert any(item["key"] == "human-adjudication" for item in beta_release["allowed_deferred_evidence"]), ( + beta_release + ) assert benchmark_summary["public_claim_ready"] is False, benchmark_summary assert benchmark_summary["public_claim_blocker_count"] >= 3, benchmark_summary public_claim = full_payload["data"]["benchmark_reproducibility"]["public_claim"]