Add beta test release boundary

scripts/evidence_consistency_core.py

@@ -41,6 +41,9 @@ BENCHMARK_SUMMARY_KEYS = [
     "world_class_source_check_count",
     "world_class_source_pass_count",
     "world_class_source_blocked_count",
+    "beta_test_ready",
+    "beta_test_blocker_count",
+    "beta_test_deferred_evidence_count",
     "public_claim_ready",
     "public_claim_blocker_count",
 ]

scripts/render_benchmark_reproducibility.py

@@ -311,6 +311,54 @@ def public_claim_blockers(
     return blockers
 
 
+def beta_test_blockers(
+    local_reproducibility_ready: bool,
+    release_lock_ready: bool,
+    provider_evidence_complete: bool,
+) -> list[str]:
+    blockers = []
+    if not local_reproducibility_ready:
+        blockers.append("local benchmark reproducibility is incomplete")
+    if not release_lock_ready:
+        blockers.append("release lock is not clean or commit is unavailable")
+    if not provider_evidence_complete:
+        blockers.append("provider-backed model holdout source evidence is incomplete")
+    return blockers
+
+
+def beta_deferred_evidence(human_review_complete: bool, world_class_ready: bool) -> list[dict[str, str]]:
+    deferred = []
+    if not human_review_complete:
+        deferred.append(
+            {
+                "key": "human-adjudication",
+                "label": "Human blind-review adjudication",
+                "reason": "Deferred for beta/public testing; still required before superiority, fully-reviewed, or world-class claims.",
+            }
+        )
+    if not world_class_ready:
+        deferred.extend(
+            [
+                {
+                    "key": "provider-ledger-review",
+                    "label": "Independent provider ledger review",
+                    "reason": "Provider source evidence may be complete, but independent ledger acceptance remains formal-claim evidence.",
+                },
+                {
+                    "key": "native-permission-enforcement",
+                    "label": "Native permission enforcement evidence",
+                    "reason": "Native enforcement proof is deferred for beta/public testing and remains required for world-class claims.",
+                },
+                {
+                    "key": "native-client-telemetry",
+                    "label": "Real client telemetry evidence",
+                    "reason": "Real client telemetry is deferred for beta/public testing and remains required for world-class claims.",
+                },
+            ]
+        )
+    return deferred
+
+
 def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
     reports = skill_dir / "reports"
     output_quality = load_json(reports / "output_quality_scorecard.json")
@@ -364,10 +412,18 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
         world_class_source_blocked_count,
     )
     public_claim_ready = not claim_blockers
+    beta_blockers = beta_test_blockers(
+        local_reproducibility_ready,
+        release_lock["ready"],
+        provider_evidence_complete,
+    )
+    beta_deferred = beta_deferred_evidence(human_review_complete, world_class_ready)
+    beta_test_ready = not beta_blockers
     limitations = [
         "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
         "Pending blind-review decisions are visible but do not count as human adjudication.",
         "World-class readiness remains false until external and human evidence gaps close.",
+        "Beta/public testing may proceed without human blind-review only when wording avoids superiority, fully-reviewed, or world-class claims.",
     ]
     if provider_evidence_complete:
         limitations.insert(
@@ -411,6 +467,9 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
             "world_class_source_check_count": world_class_source_check_count,
             "world_class_source_pass_count": world_class_source_pass_count,
             "world_class_source_blocked_count": world_class_source_blocked_count,
+            "beta_test_ready": beta_test_ready,
+            "beta_test_blocker_count": len(beta_blockers),
+            "beta_test_deferred_evidence_count": len(beta_deferred),
             "public_claim_ready": public_claim_ready,
             "public_claim_blocker_count": len(claim_blockers),
             "working_tree_dirty": status.get("dirty"),
@@ -420,6 +479,14 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
             "generated_tree_dirty": status.get("generated_dirty"),
             "generated_changed_file_count": status.get("generated_changed_file_count"),
         },
+        "beta_test_release": {
+            "ready": beta_test_ready,
+            "scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
+            "blockers": beta_blockers,
+            "allowed_deferred_evidence": beta_deferred,
+            "policy": "Human blind-review, native permission enforcement, real client telemetry, and ledger acceptance may be deferred for beta/public testing, but public claims must remain blocked until those evidence entries are accepted.",
+            "required_wording": "Use beta, public test, or technical preview wording; do not claim world-class readiness, fully reviewed quality, or proven superiority over baseline.",
+        },
         "public_claim": {
             "ready": public_claim_ready,
             "scope": "public benchmark or world-class readiness claim",
@@ -473,6 +540,9 @@ def render_markdown(report: dict[str, Any]) -> str:
         f"- human review complete: `{str(summary['human_review_complete']).lower()}`",
         f"- world-class ready: `{str(summary['world_class_ready']).lower()}`",
         f"- world-class source checks: `{summary.get('world_class_source_pass_count', 0)}` pass / `{summary.get('world_class_source_check_count', 0)}` total; `{summary.get('world_class_source_blocked_count', 0)}` blocked",
+        f"- beta test ready: `{str(summary['beta_test_ready']).lower()}`",
+        f"- beta test blockers: `{summary['beta_test_blocker_count']}`",
+        f"- beta deferred evidence: `{summary['beta_test_deferred_evidence_count']}`",
         f"- public claim ready: `{str(summary['public_claim_ready']).lower()}`",
         f"- public claim blockers: `{summary['public_claim_blocker_count']}`",
         f"- changed files at generation: `{summary.get('changed_file_count')}`",
@@ -481,15 +551,49 @@ def render_markdown(report: dict[str, Any]) -> str:
         "",
         "This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.",
         "",
-        "## Public Claim Boundary",
-        "",
-        f"- ready: `{str(report.get('public_claim', {}).get('ready')).lower()}`",
-        f"- scope: {report.get('public_claim', {}).get('scope', 'public benchmark claim')}",
-        f"- policy: {report.get('public_claim', {}).get('policy', '')}",
-        "",
-        "| Blocker |",
-        "| --- |",
     ]
+    beta_release = report.get("beta_test_release", {})
+    lines.extend(
+        [
+            "## Beta Test Boundary",
+            "",
+            f"- ready: `{str(beta_release.get('ready')).lower()}`",
+            f"- scope: {beta_release.get('scope', 'beta/public test release')}",
+            f"- policy: {beta_release.get('policy', '')}",
+            f"- required wording: {beta_release.get('required_wording', '')}",
+            "",
+            "| Blocker |",
+            "| --- |",
+        ]
+    )
+    beta_blockers = beta_release.get("blockers", [])
+    if beta_blockers:
+        lines.extend(f"| {item} |" for item in beta_blockers)
+    else:
+        lines.append("| none |")
+    lines.extend(["", "| Deferred evidence | Reason |", "| --- | --- |"])
+    deferred = beta_release.get("allowed_deferred_evidence", [])
+    if deferred:
+        lines.extend(f"| `{item.get('key', '')}` | {item.get('reason', '')} |" for item in deferred)
+    else:
+        lines.append("| none | none |")
+    lines.extend(
+        [
+            "",
+            "## Public Claim Boundary",
+            "",
+        ]
+    )
+    lines.extend(
+        [
+            f"- ready: `{str(report.get('public_claim', {}).get('ready')).lower()}`",
+            f"- scope: {report.get('public_claim', {}).get('scope', 'public benchmark claim')}",
+            f"- policy: {report.get('public_claim', {}).get('policy', '')}",
+            "",
+            "| Blocker |",
+            "| --- |",
+        ]
+    )
     blockers = report.get("public_claim", {}).get("blockers", [])
     if blockers:
         lines.extend(f"| {item} |" for item in blockers)

scripts/render_evidence_consistency.py

@@ -450,6 +450,46 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
             paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]],
             detail="Benchmark reproducibility must not overstate public claim readiness.",
         )
+        expected_beta_ready = (
+            bool(benchmark_summary.get("reproducibility_ready"))
+            and bool(benchmark_summary.get("release_lock_ready"))
+            and bool(benchmark_summary.get("provider_evidence_complete"))
+        )
+        beta_boundary = {
+            "beta_test_ready": expected_beta_ready,
+            "public_claim_ready": ledger_summary.get("ready_to_claim_world_class"),
+            "human_review_complete": False,
+            "beta_release_ready": expected_beta_ready,
+            "beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
+            "deferred_human_review": True,
+        }
+        beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {}
+        deferred_keys = {
+            str(item.get("key", ""))
+            for item in beta_release.get("allowed_deferred_evidence", [])
+            if isinstance(item, dict)
+        }
+        actual_beta_boundary = {
+            "beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None,
+            "public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None,
+            "human_review_complete": benchmark_summary.get("human_review_complete")
+            if isinstance(benchmark_summary, dict)
+            else None,
+            "beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None,
+            "beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None,
+            "deferred_human_review": "human-adjudication" in deferred_keys,
+        }
+        compare_values(
+            checks,
+            key="benchmark-beta-public-claim-split",
+            label="Benchmark separates beta testing from public claims",
+            expected=beta_boundary,
+            actual=actual_beta_boundary,
+            paths=[REQUIRED_REPORTS["benchmark"], REQUIRED_REPORTS["world_class_ledger"]],
+            detail=(
+                "Beta/public testing may proceed with human blind review deferred, but public world-class or superiority claims must remain blocked."
+            ),
+        )
         preflight_boundary = {
             "pending_count": ledger_summary.get("pending_count"),
             "source_check_count": ledger_summary.get("source_check_count"),

tests/verify_benchmark_reproducibility.py

@@ -173,6 +173,27 @@ def main() -> None:
         + payload["summary"]["world_class_source_blocked_count"]
         == payload["summary"]["world_class_source_check_count"]
     ), payload
+    expected_beta_ready = (
+        payload["summary"]["reproducibility_ready"]
+        and payload["summary"]["release_lock_ready"]
+        and payload["summary"]["provider_evidence_complete"]
+    )
+    assert payload["summary"]["beta_test_ready"] is expected_beta_ready, payload
+    assert payload["summary"]["beta_test_blocker_count"] == len(payload["beta_test_release"]["blockers"]), payload
+    assert payload["summary"]["beta_test_deferred_evidence_count"] == len(
+        payload["beta_test_release"]["allowed_deferred_evidence"]
+    ), payload
+    assert payload["beta_test_release"]["ready"] is expected_beta_ready, payload["beta_test_release"]
+    assert "beta/public test release" in payload["beta_test_release"]["scope"], payload["beta_test_release"]
+    assert "Human blind-review" in payload["beta_test_release"]["policy"], payload["beta_test_release"]
+    assert "do not claim world-class" in payload["beta_test_release"]["required_wording"], payload[
+        "beta_test_release"
+    ]
+    deferred_keys = {item["key"] for item in payload["beta_test_release"]["allowed_deferred_evidence"]}
+    assert "human-adjudication" in deferred_keys, payload["beta_test_release"]
+    assert not any("human blind-review" in item for item in payload["beta_test_release"]["blockers"]), payload[
+        "beta_test_release"
+    ]
     assert payload["summary"]["public_claim_ready"] is False, payload
     minimum_blockers = 3 if payload["summary"]["release_lock_ready"] else 4
     assert payload["summary"]["public_claim_blocker_count"] >= minimum_blockers, payload
@@ -221,6 +242,10 @@ def main() -> None:
     assert "source changed files at generation" in markdown, markdown
     assert "generated changed files at generation" in markdown, markdown
     assert "world-class source checks" in markdown, markdown
+    assert "beta test ready" in markdown, markdown
+    assert "## Beta Test Boundary" in markdown, markdown
+    assert "human-adjudication" in markdown, markdown
+    assert "do not claim world-class" in markdown, markdown
     assert "public claim ready: `false`" in markdown, markdown
     assert "## Public Claim Boundary" in markdown, markdown
     assert "provider-backed model holdout evidence is incomplete" not in markdown, markdown

tests/verify_evidence_consistency.py

@@ -196,6 +196,13 @@ def main() -> None:
     assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"]
     assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"]
     assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"]
+    assert checks["benchmark-beta-public-claim-split"]["status"] == "pass", checks[
+        "benchmark-beta-public-claim-split"
+    ]
+    beta_split = checks["benchmark-beta-public-claim-split"]["actual"]
+    assert beta_split["public_claim_ready"] is False, beta_split
+    assert beta_split["human_review_complete"] is False, beta_split
+    assert beta_split["deferred_human_review"] is True, beta_split
     assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"]
     assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[
         "preflight-submission-kit-handoff"

tests/verify_review_studio.py

@@ -212,6 +212,19 @@ def main() -> None:
     assert benchmark_summary["reproducibility_ready"] is True, benchmark_summary
     assert benchmark_summary["release_lock_ready"] == (benchmark_summary["source_tree_dirty"] is False), benchmark_summary
     assert "generated_tree_dirty" in benchmark_summary, benchmark_summary
+    expected_beta_ready = (
+        benchmark_summary["reproducibility_ready"]
+        and benchmark_summary["release_lock_ready"]
+        and benchmark_summary["provider_evidence_complete"]
+    )
+    assert benchmark_summary["beta_test_ready"] is expected_beta_ready, benchmark_summary
+    assert benchmark_summary["beta_test_deferred_evidence_count"] >= 1, benchmark_summary
+    beta_release = full_payload["data"]["benchmark_reproducibility"]["beta_test_release"]
+    assert beta_release["ready"] is expected_beta_ready, beta_release
+    assert not any("human blind-review" in item for item in beta_release["blockers"]), beta_release
+    assert any(item["key"] == "human-adjudication" for item in beta_release["allowed_deferred_evidence"]), (
+        beta_release
+    )
     assert benchmark_summary["public_claim_ready"] is False, benchmark_summary
     assert benchmark_summary["public_claim_blocker_count"] >= 3, benchmark_summary
     public_claim = full_payload["data"]["benchmark_reproducibility"]["public_claim"]