diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index 0857a20..c167415 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -11,6 +11,7 @@ from typing import Any from render_world_class_evidence_intake import build_intake from world_class_evidence_contract import DISALLOWED_REAL_ARTIFACTS +from world_class_source_checks import build_source_checklist, summarize_source_checklist ROOT = Path(__file__).resolve().parent.parent @@ -197,68 +198,6 @@ def build_artifact_checklist(skill_dir: Path, items: list[dict[str, Any]]) -> li return rows -SOURCE_CHECK_SPECS = { - "provider-holdout": [ - ("Provider model run", "model_executed_count", ">0", "Run provider-backed output-exec with real credentials."), - ("Timing observed", "timing_observed_count", ">0", "Provider execution should record timing metadata."), - ("Token usage observed", "token_observed_count", ">0", "Provider execution should return non-estimated token usage."), - ], - "human-adjudication": [ - ("Review pairs exist", "pair_count", ">0", "Generate the blind A/B review pack."), - ("No pending decisions", "pending_count", "==0", "Record a reviewer choice for every pair."), - ("Judgments complete", "judgment_count", "==pair_count", "Every pair needs one valid human judgment."), - ("No invalid decisions", "invalid_decision_count", "==0", "Fix malformed winner/confidence entries."), - ], - "native-permission-enforcement": [ - ("Native enforcement", "native_enforcement_count", ">0", "Collect real target-client or external runtime guard proof."), - ("Probe failures", "failure_count", "==0", "Runtime permission probes must stay clean."), - ("Installer support", "installer_enforcement_ready", "true", "Installer enforcement is supporting evidence, not native proof."), - ], - "native-client-telemetry": [ - ("External events", "external_source_events", ">0", "Import at least one metadata-only event from a real client."), - ("Adoption sample", "adoption_sample_count", ">0", "Telemetry must include adoption outcome evidence."), - ("Raw content blocked", "raw_content_allowed", "false", "Telemetry must stay metadata-only."), - ], -} - - -def source_check_passed(actual: Any, expected: str, observed_state: dict[str, Any]) -> bool: - if expected == ">0": - return isinstance(actual, (int, float)) and actual > 0 - if expected == "==0": - return actual == 0 - if expected == "true": - return actual is True - if expected == "false": - return actual is False - if expected == "==pair_count": - return actual == observed_state.get("pair_count") and isinstance(actual, (int, float)) and actual > 0 - return False - - -def build_source_checklist(items: list[dict[str, Any]]) -> list[dict[str, Any]]: - rows: list[dict[str, Any]] = [] - for item in items: - key = str(item.get("evidence_key", "")) - observed_state = item.get("observed_state", {}) if isinstance(item.get("observed_state", {}), dict) else {} - for label, field, expected, next_action in SOURCE_CHECK_SPECS.get(key, []): - actual = observed_state.get(field) - passed = source_check_passed(actual, expected, observed_state) - rows.append( - { - "evidence_key": key, - "label": label, - "field": field, - "expected": expected, - "actual": actual, - "status": "pass" if passed else "blocked", - "source_accepted": item.get("source_accepted") is True, - "next_action": next_action, - } - ) - return rows - - def render_readme(report: dict[str, Any]) -> str: commands = report["commands"] lines = [ @@ -588,8 +527,7 @@ def build_submission_kit( artifact_ready_count = sum(1 for item in artifact_checklist if item.get("artifact_ref_ready")) artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready")) artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required")) - source_pass_count = sum(1 for item in source_checklist if item.get("status") == "pass") - source_blocked_count = sum(1 for item in source_checklist if item.get("status") != "pass") + source_summary = summarize_source_checklist(source_checklist) ok = not unknown_keys and skipped_count == 0 report = { "schema_version": "1.0", @@ -608,9 +546,7 @@ def build_submission_kit( "artifact_ready_count": artifact_ready_count, "artifact_missing_count": artifact_missing_count, "artifact_glob_expansion_count": artifact_glob_count, - "source_check_count": len(source_checklist), - "source_pass_count": source_pass_count, - "source_blocked_count": source_blocked_count, + **source_summary, "drafts_count_as_evidence": False, "ledger_counts_submission_as_completion": False, "decision": "submission-kit-ready" if ok else "fix-submission-kit", diff --git a/scripts/render_world_class_operator_runbook.py b/scripts/render_world_class_operator_runbook.py index 5fb3fe9..b263a9a 100644 --- a/scripts/render_world_class_operator_runbook.py +++ b/scripts/render_world_class_operator_runbook.py @@ -73,6 +73,7 @@ def build_runbook_item( }, "evidence_artifacts": entry.get("evidence_artifacts", []), "observed_state": entry.get("observed_state", {}), + "source_checklist": review_item.get("source_checklist", []), "submission_state": entry.get("submission_state", {}), "anti_overclaim": entry.get("anti_overclaim", {}), } @@ -104,6 +105,9 @@ def build_operator_runbook(skill_dir: Path, generated_at: str, submissions_dir: "ready_for_ledger_review_count": review_summary.get("ready_for_ledger_review_count", 0), "valid_packet_source_incomplete_count": review_summary.get("valid_packet_source_incomplete_count", 0), "invalid_submission_count": review_summary.get("invalid_submission_count", 0), + "source_check_count": review_summary.get("source_check_count", 0), + "source_pass_count": review_summary.get("source_pass_count", 0), + "source_blocked_count": review_summary.get("source_blocked_count", 0), "ready_to_claim_world_class": summary.get("ready_to_claim_world_class") is True, "runbook_counts_as_completion": False, "decision": "ready-for-completion-audit" if summary.get("ready_to_claim_world_class") is True else "collect-evidence", @@ -194,6 +198,23 @@ def render_markdown(report: dict[str, Any]) -> str: lines.extend(list_lines(item["must_collect"].get("privacy_contract", []), "No privacy contract listed.")) lines.extend(["", "### Evidence Artifacts", ""]) lines.extend(list_lines(item.get("evidence_artifacts", []), "No evidence artifacts listed.")) + lines.extend( + [ + "", + "### Source Evidence Snapshot", + "", + "| Check | Current | Expected | Status |", + "| --- | --- | --- | --- |", + ] + ) + source_rows = item.get("source_checklist", []) + if source_rows: + for row in source_rows: + lines.append( + f"| {row['label']} | `{row['actual']}` | `{row['expected']}` | `{row['status']}` |" + ) + else: + lines.append("| No source checks listed. | `n/a` | `n/a` | `n/a` |") lines.extend( [ "", @@ -213,6 +234,23 @@ def html_list(values: list[Any], empty: str) -> str: return "".join(f"
  • {html_text(value)}
  • " for value in values) +def html_source_checks(rows: list[dict[str, Any]]) -> str: + if not rows: + return "

    No source checks listed.

    " + items = [] + for row in rows: + items.append( + "
  • " + f"{html_text(row.get('label', ''))}" + f"{html_text(row.get('field', ''))}: {html_text(row.get('actual', ''))} / {html_text(row.get('expected', ''))}" + f"{html_text(row.get('next_action', ''))}" + "
  • " + ) + return "" + + def render_html_item(item: dict[str, Any]) -> str: commands = "".join( f"
  • {html_text(label.replace('_', ' '))}{html_text(command)}
  • " @@ -234,6 +272,7 @@ def render_html_item(item: dict[str, Any]) -> str:

    Success Checks

    Privacy

    +

    Source Evidence Snapshot

    {html_source_checks(item.get('source_checklist', []))}
    """.strip() @@ -244,6 +283,7 @@ def render_html(report: dict[str, Any]) -> str: ("Pending", summary["pending_count"]), ("Awaiting", summary["awaiting_submission_count"]), ("Ready", summary["ready_for_ledger_review_count"]), + ("Source", f"{summary.get('source_pass_count', 0)}/{summary.get('source_check_count', 0)}"), ("Invalid", summary["invalid_submission_count"]), ] stat_html = "".join(f"
    {html_text(label)}{html_text(value)}
    " for label, value in stats) @@ -272,7 +312,7 @@ def render_html(report: dict[str, Any]) -> str: h3 {{ margin:4px 0 10px; font-size:22px; }} h4 {{ margin:0 0 8px; font-size:16px; }} .lede {{ max-width:800px; color:var(--muted); font-size:20px; }} - .stats {{ display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin-top:26px; }} + .stats {{ display:grid; grid-template-columns:repeat(5, minmax(0,1fr)); gap:12px; margin-top:26px; }} .stats article, .panel, .item-card {{ border:1px solid var(--line); border-radius:8px; background:#fff; }} .stats article {{ padding:16px; }} .stats span, .item-card span, .muted {{ color:var(--muted); }} @@ -292,6 +332,11 @@ def render_html(report: dict[str, Any]) -> str: .mini-grid {{ display:grid; grid-template-columns:repeat(3, minmax(0,1fr)); gap:12px; margin-top:14px; }} .mini-grid section, .panel {{ background:var(--soft); border-radius:8px; padding:14px; min-width:0; }} .mini-grid li, .panel li {{ overflow-wrap:anywhere; }} + .source-panel {{ background:var(--soft); border-radius:8px; padding:14px; min-width:0; }} + .source-checks {{ list-style:none; padding:0; margin:0; display:grid; gap:8px; }} + .source-check {{ border-top:1px solid var(--line); padding-top:8px; display:grid; gap:3px; }} + .source-check span {{ color:var(--ink); }} + .source-check code, .source-check small {{ overflow-wrap:anywhere; }} @media (max-width:820px) {{ .stats, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} diff --git a/scripts/render_world_class_submission_review.py b/scripts/render_world_class_submission_review.py index cd0c235..409e142 100644 --- a/scripts/render_world_class_submission_review.py +++ b/scripts/render_world_class_submission_review.py @@ -7,6 +7,7 @@ from typing import Any from render_world_class_evidence_intake import build_intake from render_world_class_evidence_ledger import build_ledger +from world_class_source_checks import build_source_checklist, summarize_source_checklist ROOT = Path(__file__).resolve().parent.parent @@ -55,6 +56,8 @@ def build_item(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> d state, reason = review_state(entry, intake_result) submission = entry.get("submission_state", {}) if isinstance(entry.get("submission_state", {}), dict) else {} observed = entry.get("observed_state", {}) if isinstance(entry.get("observed_state", {}), dict) else {} + source_checklist = build_source_checklist([entry]) + source_summary = summarize_source_checklist(source_checklist) return { "evidence_key": entry.get("key", ""), "label": entry.get("label", entry.get("key", "")), @@ -72,6 +75,8 @@ def build_item(entry: dict[str, Any], intake_result: dict[str, Any] | None) -> d "artifact_ref_count": submission.get("artifact_ref_count", 0), "intake_errors": intake_result.get("errors", []) if intake_result else [], "observed_state": observed, + "source_checklist": source_checklist, + **source_summary, "success_checks": entry.get("success_checks", []), "privacy_contract": entry.get("privacy_contract", []), "next_action": entry.get("next_action", ""), @@ -96,6 +101,8 @@ def build_submission_review(skill_dir: Path, generated_at: str, submissions_dir: accepted_count = counts.get("accepted", 0) source_incomplete_count = counts.get("source-evidence-incomplete", 0) awaiting_count = counts.get("awaiting-submission", 0) + source_rows = [row for item in items for row in item.get("source_checklist", [])] + source_summary = summarize_source_checklist(source_rows) if accepted_count == len(items) and items: decision = "ledger-complete" elif invalid_count: @@ -120,6 +127,7 @@ def build_submission_review(skill_dir: Path, generated_at: str, submissions_dir: "fix_submission_count": counts.get("fix-submission", 0), "unmatched_submission_count": len(unmatched), "invalid_submission_count": invalid_count, + **source_summary, "ready_to_claim_world_class": ledger.get("summary", {}).get("ready_to_claim_world_class") is True, "review_counts_submission_as_completion": False, "decision": decision, @@ -202,7 +210,17 @@ def render_markdown(report: dict[str, Any]) -> str: "", "#### Source Checks", "", - *render_list(item.get("success_checks", []), "No source checks listed."), + *render_list( + [ + f"{row['label']}: {row['actual']} / {row['expected']} => {row['status']}" + for row in item.get("source_checklist", []) + ], + "No source checks listed.", + ), + "", + "#### Completion Assertions", + "", + *render_list(item.get("success_checks", []), "No completion assertions listed."), "", "#### Intake Errors", "", diff --git a/scripts/review_studio_layout.py b/scripts/review_studio_layout.py index be6f27e..58ffbf4 100644 --- a/scripts/review_studio_layout.py +++ b/scripts/review_studio_layout.py @@ -253,6 +253,44 @@ def review_studio_css() -> str: margin: 0; padding-left: 18px; } + .world-source-panel { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-source-panel h4 { + margin: 0 0 8px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-source-checks { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 8px; + } + .world-source-check { + display: grid; + grid-template-columns: minmax(0, .7fr) minmax(0, .6fr) minmax(0, 1fr); + gap: 10px; + align-items: start; + min-width: 0; + padding: 9px 0 0; + border-top: 1px solid var(--line); + } + .world-source-check span { + color: var(--ink); + font-size: 14px; + } + .world-source-check code, + .world-source-check small { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .world-source-check.pass { border-left: 3px solid var(--pass); padding-left: 8px; } + .world-source-check.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } .output-review-grid { display: grid; grid-template-columns: minmax(0, 1fr); @@ -515,7 +553,7 @@ def review_studio_css() -> str: font-size: 13px; } @media (max-width: 980px) { - .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .output-review-grid, .output-review-steps, .world-evidence-grid, .world-evidence-columns, .world-intake-grid, .world-intake-steps, .waiver-candidate-grid, .waiver-card dl, .kv-grid { grid-template-columns: 1fr; } + .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .output-review-grid, .output-review-steps, .world-evidence-grid, .world-evidence-columns, .world-source-check, .world-intake-grid, .world-intake-steps, .waiver-candidate-grid, .waiver-card dl, .kv-grid { grid-template-columns: 1fr; } main { padding: 32px 18px 60px; } nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; } nav a { flex: 0 0 auto; } diff --git a/scripts/review_studio_world_class.py b/scripts/review_studio_world_class.py index 31045c6..d708938 100644 --- a/scripts/review_studio_world_class.py +++ b/scripts/review_studio_world_class.py @@ -4,6 +4,8 @@ import html from typing import Any +from world_class_source_checks import build_source_checklist + SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by render_review_studio.py to render world-class evidence cards." @@ -15,6 +17,25 @@ def render_inline_list(items: list[Any], empty_label: str) -> str: return "" +def render_source_checks(entry: dict[str, Any]) -> str: + rows = build_source_checklist([entry]) + if not rows: + return "

    暂无源证据检查。

    " + items = [] + for row in rows: + status = str(row.get("status", "blocked")) + items.append( + "
  • " + f"{html.escape(str(row.get('label', '')))}" + f"{html.escape(str(row.get('field', '')))}: {html.escape(str(row.get('actual', '')))} / {html.escape(str(row.get('expected', '')))}" + f"{html.escape(str(row.get('next_action', '')))}" + "
  • " + ) + return "" + + def render_world_class_evidence_entries(ledger: dict[str, Any]) -> str: entries = ledger.get("entries", []) if isinstance(ledger, dict) else [] if not entries: @@ -52,6 +73,9 @@ def render_world_class_evidence_entries(ledger: dict[str, Any]) -> str: + render_inline_list(entry.get("privacy_contract", []), "暂无隐私约束。") + "" "" + "

    源证据检查

    " + + render_source_checks(entry) + + "
    " "" ) return "
    " + "".join(cards) + "
    " diff --git a/scripts/world_class_source_checks.py b/scripts/world_class_source_checks.py new file mode 100644 index 0000000..38ad77b --- /dev/null +++ b/scripts/world_class_source_checks.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Shared source-evidence checks for world-class evidence workflows.""" + +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to keep source-evidence readiness checks consistent." + + +SOURCE_CHECK_SPECS = { + "provider-holdout": [ + ("Provider model run", "model_executed_count", ">0", "Run provider-backed output-exec with real credentials."), + ("Timing observed", "timing_observed_count", ">0", "Provider execution should record timing metadata."), + ("Token usage observed", "token_observed_count", ">0", "Provider execution should return non-estimated token usage."), + ], + "human-adjudication": [ + ("Review pairs exist", "pair_count", ">0", "Generate the blind A/B review pack."), + ("No pending decisions", "pending_count", "==0", "Record a reviewer choice for every pair."), + ("Judgments complete", "judgment_count", "==pair_count", "Every pair needs one valid human judgment."), + ("No invalid decisions", "invalid_decision_count", "==0", "Fix malformed winner/confidence entries."), + ], + "native-permission-enforcement": [ + ("Native enforcement", "native_enforcement_count", ">0", "Collect real target-client or external runtime guard proof."), + ("Probe failures", "failure_count", "==0", "Runtime permission probes must stay clean."), + ("Installer support", "installer_enforcement_ready", "true", "Installer enforcement is supporting evidence, not native proof."), + ], + "native-client-telemetry": [ + ("External events", "external_source_events", ">0", "Import at least one metadata-only event from a real client."), + ("Adoption sample", "adoption_sample_count", ">0", "Telemetry must include adoption outcome evidence."), + ("Raw content blocked", "raw_content_allowed", "false", "Telemetry must stay metadata-only."), + ], +} + + +def source_check_passed(actual: Any, expected: str, observed_state: dict[str, Any]) -> bool: + if expected == ">0": + return isinstance(actual, (int, float)) and actual > 0 + if expected == "==0": + return actual == 0 + if expected == "true": + return actual is True + if expected == "false": + return actual is False + if expected == "==pair_count": + return actual == observed_state.get("pair_count") and isinstance(actual, (int, float)) and actual > 0 + return False + + +def item_evidence_key(item: dict[str, Any]) -> str: + return str(item.get("evidence_key") or item.get("key") or "") + + +def item_observed_state(item: dict[str, Any]) -> dict[str, Any]: + observed = item.get("observed_state", {}) + return observed if isinstance(observed, dict) else {} + + +def item_source_accepted(item: dict[str, Any], observed_state: dict[str, Any]) -> bool: + return item.get("source_accepted") is True or observed_state.get("accepted") is True + + +def build_source_checklist(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for item in items: + key = item_evidence_key(item) + observed_state = item_observed_state(item) + for label, field, expected, next_action in SOURCE_CHECK_SPECS.get(key, []): + actual = observed_state.get(field) + passed = source_check_passed(actual, expected, observed_state) + rows.append( + { + "evidence_key": key, + "label": label, + "field": field, + "expected": expected, + "actual": actual, + "status": "pass" if passed else "blocked", + "source_accepted": item_source_accepted(item, observed_state), + "next_action": next_action, + } + ) + return rows + + +def summarize_source_checklist(rows: list[dict[str, Any]]) -> dict[str, int]: + pass_count = sum(1 for item in rows if item.get("status") == "pass") + blocked_count = sum(1 for item in rows if item.get("status") != "pass") + return { + "source_check_count": len(rows), + "source_pass_count": pass_count, + "source_blocked_count": blocked_count, + } diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 6ee762f..ff379ef 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -481,6 +481,8 @@ def main() -> None: assert full_payload["data"]["world_class_submission_review"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_submission_review"] assert full_payload["data"]["world_class_submission_review"]["summary"]["review_counts_submission_as_completion"] is False, full_payload["data"]["world_class_submission_review"] assert full_payload["data"]["world_class_submission_review"]["summary"]["awaiting_submission_count"] == 4, full_payload["data"]["world_class_submission_review"] + assert full_payload["data"]["world_class_submission_review"]["summary"]["source_check_count"] >= 13, full_payload["data"]["world_class_submission_review"] + assert full_payload["data"]["world_class_submission_review"]["summary"]["source_blocked_count"] >= 6, full_payload["data"]["world_class_submission_review"] assert full_payload["data"]["world_class_operator_runbook"]["summary"]["decision"] == "collect-evidence", full_payload["data"]["world_class_operator_runbook"] assert full_payload["data"]["world_class_operator_runbook"]["summary"]["runbook_counts_as_completion"] is False, full_payload["data"]["world_class_operator_runbook"] assert full_payload["data"]["world_class_operator_runbook"]["summary"]["awaiting_submission_count"] == 4, full_payload["data"]["world_class_operator_runbook"] @@ -589,6 +591,11 @@ def main() -> None: assert "隐私约束" in html, html assert "reports/output_execution_runs.json summary.model_executed_count > 0" in html, html assert "计划、metadata fallback、待评审和本地命令不会被当成完成证据" in html, html + assert "源证据检查" in html, html + assert "world-source-checks" in html, html + assert "Provider model run" in html, html + assert "model_executed_count: 0 / >0" in html, html + assert "Token usage observed" in html, html assert "蓝图覆盖" in html, html assert "本地蓝图" in html, html assert "public world-class 仍以 world-class evidence ledger" in html, html diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py index 22d9941..b3ac62c 100644 --- a/tests/verify_trust_check.py +++ b/tests/verify_trust_check.py @@ -96,6 +96,7 @@ def main() -> None: "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/world_class_evidence_contract.py", + "scripts/world_class_source_checks.py", "scripts/yao_cli_config.py", "scripts/yao_cli_parser.py", "scripts/yao_cli_telemetry.py", @@ -111,6 +112,7 @@ def main() -> None: assert "skill_report_metrics.py" not in warning_text, payload["warnings"] assert "skill_report_model.py" not in warning_text, payload["warnings"] assert "world_class_evidence_contract.py" not in warning_text, payload["warnings"] + assert "world_class_source_checks.py" not in warning_text, payload["warnings"] assert "yao_cli_config.py" not in warning_text, payload["warnings"] assert "yao_cli_parser.py" not in warning_text, payload["warnings"] assert "yao_cli_telemetry.py" not in warning_text, payload["warnings"] diff --git a/tests/verify_world_class_operator_runbook.py b/tests/verify_world_class_operator_runbook.py index e1fd8a9..abc277b 100644 --- a/tests/verify_world_class_operator_runbook.py +++ b/tests/verify_world_class_operator_runbook.py @@ -105,6 +105,9 @@ def main() -> None: assert summary["ready_for_ledger_review_count"] == 0, summary assert summary["valid_packet_source_incomplete_count"] == 0, summary assert summary["invalid_submission_count"] == 0, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary assert summary["ready_to_claim_world_class"] is False, summary assert summary["runbook_counts_as_completion"] is False, summary items = {item["evidence_key"]: item for item in payload["items"]} @@ -124,14 +127,22 @@ def main() -> None: assert "world-class-claim-guard" in provider["commands"]["guard_claim"], provider assert "provider-backed model run" in provider["must_collect"]["provenance_requirements"], provider assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider["must_collect"]["success_checks"], provider + provider_source = {item["field"]: item for item in provider["source_checklist"]} + assert provider_source["model_executed_count"]["status"] == "blocked", provider_source + assert provider_source["timing_observed_count"]["status"] == "pass", provider_source + assert provider_source["token_observed_count"]["status"] == "blocked", provider_source markdown = output_md.read_text(encoding="utf-8") assert "World-Class Operator Runbook" in markdown, markdown assert "runbook counts as completion: `false`" in markdown, markdown assert "Valid intake means ready for submission review; ledger review still requires passing source evidence." in markdown, markdown + assert "Source Evidence Snapshot" in markdown, markdown + assert "| Provider model run | `0` | `>0` | `blocked` |" in markdown, markdown html = output_html.read_text(encoding="utf-8") assert "World-Class Operator Runbook" in html, html[:400] assert "ledger and claim guard" in html, html assert "position:sticky" in html, html + assert "Source Evidence Snapshot" in html, html + assert "model_executed_count" in html, html assert " None: assert submitted_summary["awaiting_submission_count"] == 3, submitted_summary assert submitted_summary["valid_packet_source_incomplete_count"] == 1, submitted_summary assert submitted_summary["ready_for_ledger_review_count"] == 0, submitted_summary + assert submitted_summary["source_pass_count"] + submitted_summary["source_blocked_count"] == submitted_summary["source_check_count"], submitted_summary + assert submitted_summary["source_blocked_count"] >= 6, submitted_summary assert submitted_summary["ready_to_claim_world_class"] is False, submitted_summary submitted_provider = {item["evidence_key"]: item for item in submitted["items"]}["provider-holdout"] assert submitted_provider["intake_readiness"] == "source-evidence-incomplete", submitted_provider diff --git a/tests/verify_world_class_submission_review.py b/tests/verify_world_class_submission_review.py index 885c2a8..0ebd0eb 100644 --- a/tests/verify_world_class_submission_review.py +++ b/tests/verify_world_class_submission_review.py @@ -97,14 +97,22 @@ def main() -> None: assert summary["awaiting_submission_count"] == 4, summary assert summary["ready_for_ledger_review_count"] == 0, summary assert summary["valid_packet_source_incomplete_count"] == 0, summary + assert summary["source_check_count"] >= 13, summary + assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary + assert summary["source_blocked_count"] >= 6, summary assert summary["review_counts_submission_as_completion"] is False, summary provider_item = {item["evidence_key"]: item for item in payload["items"]}["provider-holdout"] assert provider_item["review_state"] == "awaiting-submission", provider_item assert provider_item["submission_status"] == "missing", provider_item assert provider_item["source_accepted"] is False, provider_item + provider_source = {item["field"]: item for item in provider_item["source_checklist"]} + assert provider_source["model_executed_count"]["status"] == "blocked", provider_source + assert provider_source["timing_observed_count"]["status"] == "pass", provider_source + assert provider_source["token_observed_count"]["status"] == "blocked", provider_source markdown = output_md.read_text(encoding="utf-8") assert "World-Class Submission Review" in markdown, markdown assert "review counts submission as completion: `false`" in markdown, markdown + assert "Provider model run: 0 / >0 => blocked" in markdown, markdown assert "`provider-holdout`" in markdown, markdown submissions = TMP / "valid_submissions" @@ -132,6 +140,8 @@ def main() -> None: assert submitted_summary["valid_packet_source_incomplete_count"] == 1, submitted_summary assert submitted_summary["awaiting_submission_count"] == 3, submitted_summary assert submitted_summary["invalid_submission_count"] == 0, submitted_summary + assert submitted_summary["source_pass_count"] + submitted_summary["source_blocked_count"] == submitted_summary["source_check_count"], submitted_summary + assert submitted_summary["source_blocked_count"] >= 6, submitted_summary submitted_provider = {item["evidence_key"]: item for item in submitted_payload["items"]}["provider-holdout"] assert submitted_provider["review_state"] == "source-evidence-incomplete", submitted_provider assert submitted_provider["intake_status"] == "pass", submitted_provider @@ -139,6 +149,9 @@ def main() -> None: assert submitted_provider["artifact_ref_count"] == 1, submitted_provider assert submitted_provider["source_accepted"] is False, submitted_provider assert "model_executed_count" in submitted_provider["observed_state"], submitted_provider + submitted_source = {item["field"]: item for item in submitted_provider["source_checklist"]} + assert submitted_source["model_executed_count"]["status"] == "blocked", submitted_source + assert submitted_source["timing_observed_count"]["status"] == "pass", submitted_source invalid_dir = TMP / "invalid_submissions" invalid_dir.mkdir()