frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Make beta evidence deferrals ledger-driven

Browse Source
This commit is contained in:
yaojingang
2026-06-17 16:58:32 +08:00
parent 6426f9c6c9
commit e79050ed12
7 changed files with 152 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/evidence_consistency_core.py
+53
View File
@@ -217,6 +217,59 @@ def gate_by_key(review_studio: dict[str, Any], key: str) -> dict[str, Any]:
return {}
def beta_public_claim_split_values(
benchmark: dict[str, Any],
ledger: dict[str, Any],
review_studio: dict[str, Any],
) -> tuple[dict[str, Any], dict[str, Any]]:
benchmark_summary = nested(benchmark, ["summary"], {})
ledger_summary = nested(ledger, ["summary"], {})
expected_beta_ready = (
bool(benchmark_summary.get("reproducibility_ready"))
and bool(benchmark_summary.get("release_lock_ready"))
and bool(benchmark_summary.get("provider_evidence_complete"))
)
output_review_summary = nested(review_studio, ["data", "output_review_adjudication", "summary"], {})
review_pair_count = as_int(output_review_summary.get("pair_count")) if isinstance(output_review_summary, dict) else None
review_pending_count = as_int(output_review_summary.get("pending_count")) if isinstance(output_review_summary, dict) else None
expected_human_review_complete = (
review_pair_count is not None and review_pair_count > 0 and review_pending_count == 0
)
ledger_entries = ledger.get("entries", []) if isinstance(ledger, dict) else []
pending_ledger_keys = sorted(
str(entry.get("key", ""))
for entry in ledger_entries
if isinstance(entry, dict) and entry.get("status") == "pending" and str(entry.get("key", ""))
)
beta_boundary = {
"beta_test_ready": expected_beta_ready,
"public_claim_ready": ledger_summary.get("ready_to_claim_world_class"),
"human_review_complete": expected_human_review_complete,
"beta_release_ready": expected_beta_ready,
"beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
"deferred_evidence_keys": pending_ledger_keys,
"deferred_human_review": "human-adjudication" in pending_ledger_keys,
}
beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {}
deferred_keys = sorted(
str(item.get("key", ""))
for item in beta_release.get("allowed_deferred_evidence", [])
if isinstance(item, dict) and str(item.get("key", ""))
)
actual_beta_boundary = {
"beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None,
"public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None,
"human_review_complete": benchmark_summary.get("human_review_complete")
if isinstance(benchmark_summary, dict)
else None,
"beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None,
"beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None,
"deferred_evidence_keys": deferred_keys,
"deferred_human_review": "human-adjudication" in deferred_keys,
}
return beta_boundary, actual_beta_boundary
def report_contract(payload: dict[str, Any]) -> dict[str, Any]:
contract = payload.get("report_contract")
return contract if isinstance(contract, dict) else {}
scripts/render_benchmark_reproducibility.py
+38 -26
View File
@@ -326,36 +326,48 @@ def beta_test_blockers(
return blockers
def beta_deferred_evidence(human_review_complete: bool, world_class_ready: bool) -> list[dict[str, str]]:
BETA_DEFERRED_EVIDENCE = {
"provider-holdout": {
"label": "Provider holdout ledger review",
"reason": "Provider-backed source evidence exists, but formal ledger submission and reviewer acceptance are still pending before public claims.",
},
"human-adjudication": {
"label": "Human blind-review adjudication",
"reason": "Human adjudication evidence is still pending; deferred for beta/public testing and still required before superiority, fully-reviewed, or world-class claims.",
},
"native-permission-enforcement": {
"label": "Native permission enforcement evidence",
"reason": "Native enforcement proof is still pending; deferred for beta/public testing and still required before world-class claims.",
},
"native-client-telemetry": {
"label": "Real client telemetry evidence",
"reason": "Real client telemetry is still pending; deferred for beta/public testing and still required before world-class claims.",
},
}
def beta_deferred_evidence(world_class_ledger: dict[str, Any]) -> list[dict[str, str]]:
deferred = []
if not human_review_complete:
entries = world_class_ledger.get("entries", [])
if not isinstance(entries, list):
return deferred
for entry in entries:
if not isinstance(entry, dict) or entry.get("status") != "pending":
continue
key = str(entry.get("key") or "")
if not key:
continue
fallback = BETA_DEFERRED_EVIDENCE.get(key, {})
deferred.append(
{
"key": "human-adjudication",
"label": "Human blind-review adjudication",
"reason": "Deferred for beta/public testing; still required before superiority, fully-reviewed, or world-class claims.",
"key": key,
"label": str(entry.get("label") or fallback.get("label") or key),
"reason": str(
fallback.get("reason")
or "Formal evidence is still pending; deferred for beta/public testing and still required before public claims."
),
}
)
if not world_class_ready:
deferred.extend(
[
{
"key": "provider-ledger-review",
"label": "Independent provider ledger review",
"reason": "Provider source evidence may be complete, but independent ledger acceptance remains formal-claim evidence.",
},
{
"key": "native-permission-enforcement",
"label": "Native permission enforcement evidence",
"reason": "Native enforcement proof is deferred for beta/public testing and remains required for world-class claims.",
},
{
"key": "native-client-telemetry",
"label": "Real client telemetry evidence",
"reason": "Real client telemetry is deferred for beta/public testing and remains required for world-class claims.",
},
]
)
return deferred
@@ -417,7 +429,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
release_lock["ready"],
provider_evidence_complete,
)
beta_deferred = beta_deferred_evidence(human_review_complete, world_class_ready)
beta_deferred = beta_deferred_evidence(world_class_ledger)
beta_test_ready = not beta_blockers
limitations = [
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
scripts/render_evidence_consistency.py
+5 -28
View File
@@ -16,6 +16,7 @@ from evidence_consistency_core import (
REQUIRED_TEXT_REPORTS,
add_check,
as_int,
beta_public_claim_split_values,
compare_summary_keys,
compare_values,
gate_by_key,
@@ -450,35 +451,11 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["benchmark"]],
detail="Benchmark reproducibility must not overstate public claim readiness.",
)
expected_beta_ready = (
bool(benchmark_summary.get("reproducibility_ready"))
and bool(benchmark_summary.get("release_lock_ready"))
and bool(benchmark_summary.get("provider_evidence_complete"))
beta_boundary, actual_beta_boundary = beta_public_claim_split_values(
benchmark,
ledger,
review_studio,
)
beta_boundary = {
"beta_test_ready": expected_beta_ready,
"public_claim_ready": ledger_summary.get("ready_to_claim_world_class"),
"human_review_complete": False,
"beta_release_ready": expected_beta_ready,
"beta_release_scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
"deferred_human_review": True,
}
beta_release = benchmark.get("beta_test_release", {}) if isinstance(benchmark, dict) else {}
deferred_keys = {
str(item.get("key", ""))
for item in beta_release.get("allowed_deferred_evidence", [])
if isinstance(item, dict)
}
actual_beta_boundary = {
"beta_test_ready": benchmark_summary.get("beta_test_ready") if isinstance(benchmark_summary, dict) else None,
"public_claim_ready": benchmark_summary.get("public_claim_ready") if isinstance(benchmark_summary, dict) else None,
"human_review_complete": benchmark_summary.get("human_review_complete")
if isinstance(benchmark_summary, dict)
else None,
"beta_release_ready": beta_release.get("ready") if isinstance(beta_release, dict) else None,
"beta_release_scope": beta_release.get("scope") if isinstance(beta_release, dict) else None,
"deferred_human_review": "human-adjudication" in deferred_keys,
}
compare_values(
checks,
key="benchmark-beta-public-claim-split",
tests/review_studio_helpers.py
+1
View File
@@ -170,6 +170,7 @@ def prepare_review_studio_inputs(tmp_root: Path) -> None:
"render_world_class_operator_runbook.py",
"render_world_class_claim_guard.py",
"render_skill_os2_coverage.py",
"render_benchmark_reproducibility.py",
]:
allowed_returncodes = (0, 2) if script_name == "render_world_class_submission_review.py" else (0,)
run_script(script_name, ROOT, "--generated-at", "2026-06-13", allowed_returncodes=allowed_returncodes)
tests/verify_benchmark_reproducibility.py
+47
View File
@@ -9,6 +9,9 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "render_benchmark_reproducibility.py"
TMP = ROOT / "tests" / "tmp_benchmark_reproducibility"
sys.path.insert(0, str(ROOT / "scripts"))
from evidence_consistency_core import beta_public_claim_split_values # noqa: E402
from render_benchmark_reproducibility import beta_deferred_evidence # noqa: E402
def main() -> None:
@@ -189,11 +192,55 @@ def main() -> None:
assert "do not claim world-class" in payload["beta_test_release"]["required_wording"], payload[
"beta_test_release"
]
ledger_payload = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8"))
pending_ledger_keys = {
item["key"] for item in ledger_payload["entries"] if item.get("status") == "pending"
}
deferred_keys = {item["key"] for item in payload["beta_test_release"]["allowed_deferred_evidence"]}
assert deferred_keys == pending_ledger_keys, payload["beta_test_release"]
assert "human-adjudication" in deferred_keys, payload["beta_test_release"]
assert "provider-holdout" in deferred_keys, payload["beta_test_release"]
assert "provider-ledger-review" not in deferred_keys, payload["beta_test_release"]
assert not any("human blind-review" in item for item in payload["beta_test_release"]["blockers"]), payload[
"beta_test_release"
]
synthetic_deferred = beta_deferred_evidence(
{
"entries": [
{"key": "provider-holdout", "label": "Provider Holdout", "status": "accepted"},
{"key": "native-client-telemetry", "label": "Native Client Telemetry", "status": "pending"},
]
}
)
assert {item["key"] for item in synthetic_deferred} == {"native-client-telemetry"}, synthetic_deferred
expected_beta_split, actual_beta_split = beta_public_claim_split_values(
{
"summary": {
"reproducibility_ready": True,
"release_lock_ready": True,
"provider_evidence_complete": True,
"beta_test_ready": True,
"public_claim_ready": False,
"human_review_complete": True,
},
"beta_test_release": {
"ready": True,
"scope": "beta/public test release without superiority, fully-reviewed, or world-class claims",
"allowed_deferred_evidence": [{"key": "native-client-telemetry", "reason": "still pending"}],
},
},
{
"summary": {"ready_to_claim_world_class": False},
"entries": [
{"key": "human-adjudication", "status": "accepted"},
{"key": "native-client-telemetry", "status": "pending"},
],
},
{"data": {"output_review_adjudication": {"summary": {"pair_count": 5, "pending_count": 0}}}},
)
assert expected_beta_split == actual_beta_split, (expected_beta_split, actual_beta_split)
assert expected_beta_split["human_review_complete"] is True, expected_beta_split
assert expected_beta_split["deferred_human_review"] is False, expected_beta_split
assert payload["summary"]["public_claim_ready"] is False, payload
minimum_blockers = 3 if payload["summary"]["release_lock_ready"] else 4
assert payload["summary"]["public_claim_blocker_count"] >= minimum_blockers, payload
tests/verify_evidence_consistency.py
+1
View File
@@ -202,6 +202,7 @@ def main() -> None:
beta_split = checks["benchmark-beta-public-claim-split"]["actual"]
assert beta_split["public_claim_ready"] is False, beta_split
assert beta_split["human_review_complete"] is False, beta_split
assert beta_split["deferred_evidence_keys"] == ["human-adjudication", "native-client-telemetry", "native-permission-enforcement", "provider-holdout"], beta_split
assert beta_split["deferred_human_review"] is True, beta_split
assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"]
assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[
tests/verify_review_studio.py
+7
View File
@@ -222,6 +222,13 @@ def main() -> None:
beta_release = full_payload["data"]["benchmark_reproducibility"]["beta_test_release"]
assert beta_release["ready"] is expected_beta_ready, beta_release
assert not any("human blind-review" in item for item in beta_release["blockers"]), beta_release
pending_ledger_keys = {
item["key"]
for item in full_payload["data"]["world_class_evidence_ledger"]["entries"]
if item.get("status") == "pending"
}
deferred_keys = {item["key"] for item in beta_release["allowed_deferred_evidence"]}
assert deferred_keys == pending_ledger_keys, beta_release
assert any(item["key"] == "human-adjudication" for item in beta_release["allowed_deferred_evidence"]), (
beta_release
)