diff --git a/scripts/render_world_class_evidence_ledger.py b/scripts/render_world_class_evidence_ledger.py index 38a49e40..a465208a 100644 --- a/scripts/render_world_class_evidence_ledger.py +++ b/scripts/render_world_class_evidence_ledger.py @@ -7,6 +7,7 @@ from typing import Any from render_world_class_evidence_plan import build_plan from world_class_evidence_contract import load_json, load_json_with_status, rel_path, validate_payload +from world_class_source_checks import build_source_checklist, summarize_source_checklist ROOT = Path(__file__).resolve().parent.parent @@ -129,7 +130,9 @@ def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Pat def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]: state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir) submission = submission_state(skill_dir, task, submissions_dir) - source_accepted = bool(state.get("accepted")) + source_checklist = build_source_checklist([{"key": task["key"], "observed_state": state}]) + source_summary = summarize_source_checklist(source_checklist) + source_accepted = bool(source_checklist) and source_summary["source_blocked_count"] == 0 accepted = source_accepted and submission.get("status") == "submitted" return { "key": task["key"], @@ -146,6 +149,8 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> "evidence_artifacts": task["evidence_artifacts"], "privacy_contract": task["privacy_contract"], "observed_state": state, + "source_checklist": source_checklist, + **source_summary, "submission_state": submission, "anti_overclaim": { "planned_work_counts_as_evidence": False, @@ -162,6 +167,8 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") evidence_requirements = plan.get("evidence_requirements") or plan.get("tasks", []) entries = [build_entry(skill_dir, task, submissions_dir) for task in evidence_requirements] + source_rows = [row for entry in entries for row in entry.get("source_checklist", [])] + source_summary = summarize_source_checklist(source_rows) source_accepted_count = sum(1 for entry in entries if entry.get("source_accepted") is True) accepted_count = sum(1 for entry in entries if entry["status"] == "accepted") pending_count = len(entries) - accepted_count @@ -194,6 +201,7 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non "submitted_entry_count": submitted_entry_count, "missing_submission_count": missing_submission_count, "invalid_submission_count": invalid_submission_count, + **source_summary, "submitted_but_pending_count": submitted_but_pending_count, "source_accepted_without_valid_submission_count": source_accepted_without_valid_submission_count, "overclaim_guard_active": True, @@ -233,6 +241,8 @@ def render_markdown(ledger: dict[str, Any]) -> str: f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", f"- entries: `{summary['ledger_entry_count']}`", f"- source accepted: `{summary.get('source_accepted_count', 0)}`", + f"- source checks: `{summary.get('source_pass_count', 0)}` pass / `{summary.get('source_check_count', 0)}` total", + f"- source blocked: `{summary.get('source_blocked_count', 0)}`", f"- accepted: `{summary['accepted_count']}`", f"- pending: `{summary['pending_count']}`", f"- human pending: `{summary['human_pending_count']}`", @@ -262,10 +272,31 @@ def render_markdown(ledger: dict[str, Any]) -> str: lines.append(f"- objective: {entry['objective']}") lines.append(f"- source status: `{entry['source_status']}`") lines.append(f"- observed state: `{json.dumps(entry['observed_state'], ensure_ascii=False)}`") + lines.append( + f"- source checks: `{entry.get('source_pass_count', 0)}` pass / " + f"`{entry.get('source_check_count', 0)}` total" + ) lines.append(f"- submission state: `{json.dumps(entry.get('submission_state', {}), ensure_ascii=False)}`") lines.extend(["", "### Provenance Requirements", ""]) lines.extend(f"- {item}" for item in entry["provenance_requirements"]) - lines.extend(["", "### Success Checks", ""]) + lines.extend( + [ + "", + "### Source Evidence Checks", + "", + "| Check | Current | Expected | Status |", + "| --- | --- | --- | --- |", + ] + ) + source_checks = entry.get("source_checklist", []) + if source_checks: + for row in source_checks: + lines.append( + f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` |" + ) + else: + lines.append("| No source checks listed. | `n/a` | `n/a` | `n/a` |") + lines.extend(["", "### Completion Assertions", ""]) lines.extend(f"- {item}" for item in entry["success_checks"]) lines.extend(["", "### Privacy Contract", ""]) lines.extend(f"- {item}" for item in entry["privacy_contract"]) diff --git a/scripts/review_studio_world_class.py b/scripts/review_studio_world_class.py index d7089386..aaa71e7e 100644 --- a/scripts/review_studio_world_class.py +++ b/scripts/review_studio_world_class.py @@ -18,7 +18,9 @@ def render_inline_list(items: list[Any], empty_label: str) -> str: def render_source_checks(entry: dict[str, Any]) -> str: - rows = build_source_checklist([entry]) + rows = entry.get("source_checklist") + if not isinstance(rows, list): + rows = build_source_checklist([entry]) if not rows: return "

暂无源证据检查。

" items = [] diff --git a/tests/verify_world_class_evidence_ledger.py b/tests/verify_world_class_evidence_ledger.py index 4196e9fa..ac671540 100644 --- a/tests/verify_world_class_evidence_ledger.py +++ b/tests/verify_world_class_evidence_ledger.py @@ -98,6 +98,9 @@ def main() -> None: assert summary["submitted_entry_count"] == 0, summary assert summary["missing_submission_count"] == 4, summary assert summary["invalid_submission_count"] == 0, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary assert summary["submitted_but_pending_count"] == 0, summary assert summary["overclaim_guard_active"] is True, summary assert summary["ready_to_claim_world_class"] is False, summary @@ -110,6 +113,10 @@ def main() -> None: "native-client-telemetry", }, entries assert entries["provider-holdout"]["observed_state"]["model_executed_count"] == 0, entries["provider-holdout"] + provider_source = {item["field"]: item for item in entries["provider-holdout"]["source_checklist"]} + assert provider_source["model_executed_count"]["status"] == "blocked", provider_source + assert provider_source["timing_observed_count"]["status"] == "pass", provider_source + assert provider_source["token_observed_count"]["status"] == "blocked", provider_source assert entries["provider-holdout"]["submission_state"]["status"] == "missing", entries["provider-holdout"] assert entries["provider-holdout"]["submission_state"]["ledger_counts_as_completion"] is False, entries["provider-holdout"] assert entries["human-adjudication"]["observed_state"]["pending_count"] == 5, entries["human-adjudication"] @@ -128,6 +135,9 @@ def main() -> None: assert "World-Class Evidence Ledger" in markdown, markdown assert "overclaim guard active: `true`" in markdown, markdown assert "submitted entries: `0`" in markdown, markdown + assert "source checks:" in markdown, markdown + assert "Source Evidence Checks" in markdown, markdown + assert "| Provider model run | `0` | `>0` | `blocked` |" in markdown, markdown assert "`provider-holdout`" in markdown, markdown submissions = TMP / "submissions" @@ -160,6 +170,7 @@ def main() -> None: assert submitted_summary["submitted_entry_count"] == 1, submitted_summary assert submitted_summary["submitted_but_pending_count"] == 1, submitted_summary assert submitted_summary["accepted_count"] == 0, submitted_summary + assert submitted_summary["source_blocked_count"] >= 6, submitted_summary submitted_provider = {entry["key"]: entry for entry in submitted_payload["entries"]}["provider-holdout"] assert submitted_provider["status"] == "pending", submitted_provider assert submitted_provider["submission_state"]["status"] == "submitted", submitted_provider @@ -206,12 +217,14 @@ def main() -> None: assert accepted_source_summary["source_accepted_count"] == 1, accepted_source_summary assert accepted_source_summary["accepted_count"] == 0, accepted_source_summary assert accepted_source_summary["source_accepted_without_valid_submission_count"] == 1, accepted_source_summary + assert accepted_source_summary["source_check_count"] >= 13, accepted_source_summary accepted_source_provider = { entry["key"]: entry for entry in accepted_source_payload["entries"] }["provider-holdout"] assert accepted_source_provider["source_accepted"] is True, accepted_source_provider assert accepted_source_provider["status"] == "pending", accepted_source_provider assert accepted_source_provider["submission_state"]["status"] == "missing", accepted_source_provider + assert all(item["status"] == "pass" for item in accepted_source_provider["source_checklist"]), accepted_source_provider (accepted_source_skill / "reports" / "context_budget.json").write_text( json.dumps({"summary": {"unrelated": True}}, ensure_ascii=False, indent=2) + "\n",