From 44fe310092906bdef9f775132818a6a4bd59bcd0 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Sun, 14 Jun 2026 15:03:15 +0800 Subject: [PATCH] feat: surface world-class evidence runbooks --- scripts/prepare_world_class_submission_kit.py | 19 +++++++++++++++++-- scripts/render_world_class_evidence_intake.py | 5 +++++ scripts/render_world_class_evidence_ledger.py | 3 +++ .../render_world_class_operator_runbook.py | 6 ++++++ tests/verify_world_class_evidence_intake.py | 7 +++++++ tests/verify_world_class_evidence_ledger.py | 3 +++ tests/verify_world_class_operator_runbook.py | 5 +++++ 7 files changed, 46 insertions(+), 2 deletions(-) diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index 92d5f11..f7afd6f 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -224,6 +224,16 @@ def render_readme(report: dict[str, Any]) -> str: ] for item in report["files"]: lines.append(f"| `{item['evidence_key']}` | `{item['output_path']}` | `{item['status']}` |") + lines.extend(["", "## Execution Runbook", ""]) + for item in report.get("evidence_items", []): + must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + runbook = must_collect.get("runbook", []) + lines.extend(["", f"### {item.get('label', item.get('evidence_key', 'Evidence'))}", ""]) + if runbook: + for step in runbook: + lines.append(f"- `{step}`" if str(step).startswith("python3 ") or "=" in str(step) else f"- {step}") + else: + lines.append("- No source runbook listed.") lines.extend( [ "", @@ -373,6 +383,7 @@ def render_html_source_checklist(items: list[dict[str, Any]]) -> str: def render_html_item(item: dict[str, Any]) -> str: must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + runbook = must_collect.get("runbook", []) return f"""
@@ -385,6 +396,10 @@ def render_html_item(item: dict[str, Any]) -> str:
Evidence
{html_text(item.get('evidence_key', ''))}
Submission
{html_text(item.get('submission_path', ''))}
+
+

Execution Runbook

+
    {render_html_list(runbook, 'No source runbook listed.')}
+

Must Collect

@@ -467,8 +482,8 @@ def render_html(report: dict[str, Any]) -> str: .commands li {{ padding:12px; background:var(--soft); border-radius:8px; }} .commands span {{ display:block; color:var(--ink); font-weight:700; margin-bottom:4px; }} .mini-grid {{ display:grid; grid-template-columns:repeat(3, minmax(0,1fr)); gap:12px; margin-top:14px; }} - .mini-grid section {{ background:var(--soft); border-radius:8px; padding:14px; min-width:0; }} - .mini-grid li, .notice li {{ overflow-wrap:anywhere; }} + .mini-grid section, .runbook-panel {{ background:var(--soft); border-radius:8px; padding:14px; min-width:0; }} + .mini-grid li, .runbook-panel li, .notice li {{ overflow-wrap:anywhere; }} .notice {{ background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; }} .errors {{ color:var(--warn); }} @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} diff --git a/scripts/render_world_class_evidence_intake.py b/scripts/render_world_class_evidence_intake.py index 9cbb75a..328c3b9 100644 --- a/scripts/render_world_class_evidence_intake.py +++ b/scripts/render_world_class_evidence_intake.py @@ -111,6 +111,7 @@ def build_operator_checklist( }, "must_collect": { "provenance_requirements": entry.get("provenance_requirements", []), + "runbook": entry.get("runbook", []), "success_checks": entry.get("success_checks", []), "evidence_artifacts": entry.get("evidence_artifacts", []), "privacy_contract": entry.get("privacy_contract", []), @@ -270,6 +271,10 @@ def render_operator_checklist(items: list[dict[str, Any]]) -> list[str]: if values: lines.append(f"- {label}:") lines.extend(f" - {value}" for value in values) + runbook = must_collect.get("runbook", []) + if runbook: + lines.extend(["", "#### Source Runbook", ""]) + lines.extend(f"- `{value}`" if str(value).startswith("python3 ") or "=" in str(value) else f"- {value}" for value in runbook) return lines diff --git a/scripts/render_world_class_evidence_ledger.py b/scripts/render_world_class_evidence_ledger.py index a465208..06a9ec5 100644 --- a/scripts/render_world_class_evidence_ledger.py +++ b/scripts/render_world_class_evidence_ledger.py @@ -144,6 +144,7 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> "source_accepted": source_accepted, "current": task["current"], "objective": task["objective"], + "runbook": task.get("runbook", []), "provenance_requirements": PROVENANCE_REQUIREMENTS.get(task["key"], ["release reviewer evidence"]), "success_checks": task["success_checks"], "evidence_artifacts": task["evidence_artifacts"], @@ -279,6 +280,8 @@ def render_markdown(ledger: dict[str, Any]) -> str: lines.append(f"- submission state: `{json.dumps(entry.get('submission_state', {}), ensure_ascii=False)}`") lines.extend(["", "### Provenance Requirements", ""]) lines.extend(f"- {item}" for item in entry["provenance_requirements"]) + lines.extend(["", "### Source Runbook", ""]) + lines.extend(f"- `{item}`" if str(item).startswith("python3 ") or "=" in str(item) else f"- {item}" for item in entry.get("runbook", [])) lines.extend( [ "", diff --git a/scripts/render_world_class_operator_runbook.py b/scripts/render_world_class_operator_runbook.py index 456eb58..162b23f 100644 --- a/scripts/render_world_class_operator_runbook.py +++ b/scripts/render_world_class_operator_runbook.py @@ -61,6 +61,7 @@ def build_runbook_item( "source_accepted": review_item.get("source_accepted") is True, "objective": entry.get("objective", ""), "current": entry.get("current", ""), + "execution_runbook": entry.get("runbook", []), "blocking_reason": review_item.get("blocking_reason") or checklist.get("blocking_reason") or entry.get("next_action", ""), "submission_path": checklist.get("submission_path") or entry.get("submission_state", {}).get("path", ""), "template_path": checklist.get("template_path", ""), @@ -193,6 +194,10 @@ def render_markdown(report: dict[str, Any]) -> str: f"- submission: `{item['submission_path'] or 'missing'}`", f"- template: `{item['template_path'] or 'missing'}`", "", + "### Source Runbook", + "", + *list_lines(item.get("execution_runbook", []), "No source runbook listed."), + "", "### Commands", "", ] @@ -278,6 +283,7 @@ def render_html_item(item: dict[str, Any]) -> str:
Blocked
{html_text(item.get('blocked_source_check_count', 0))}
Submission
{html_text(item['submission_path'])}
+

Source Runbook

    {html_list(item.get('execution_runbook', []), 'No source runbook listed.')}

Commands

    {commands}

Must Collect

    {html_list(must_collect.get('provenance_requirements', []), 'No provenance requirements listed.')}
diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index a57b9f6..7102136 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -165,6 +165,7 @@ def main() -> None: assert checklist["provider-holdout"]["commands"]["submission_review"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"] assert checklist["provider-holdout"]["commands"]["refresh_ledger"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"] assert "provider-backed model run" in checklist["provider-holdout"]["must_collect"]["provenance_requirements"], checklist["provider-holdout"] + assert any("output-exec --provider-runner openai" in step for step in checklist["provider-holdout"]["must_collect"]["runbook"]), checklist["provider-holdout"] assert "reports/output_execution_runs.json summary.model_executed_count > 0" in checklist["provider-holdout"]["must_collect"]["success_checks"], checklist["provider-holdout"] assert checklist["provider-holdout"]["anti_overclaim"]["local_command_runner_counts_as_provider_model"] is False, checklist["provider-holdout"] assert checklist["provider-holdout"]["source_accepted"] is False, checklist["provider-holdout"] @@ -181,6 +182,8 @@ def main() -> None: assert "`python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions`" in markdown, markdown assert "`python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`" in markdown, markdown assert "`python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions`" in markdown, markdown + assert "Source Runbook" in markdown, markdown + assert "output-exec --provider-runner openai" in markdown, markdown assert "Templates and planned work do not count as accepted evidence." in markdown, markdown assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown @@ -221,6 +224,8 @@ def main() -> None: assert kit_manifest["artifacts"]["html"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/index.html"), kit_manifest["artifacts"] kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8") assert "Drafts are not accepted evidence." in kit_readme, kit_readme + assert "Execution Runbook" in kit_readme, kit_readme + assert "output-exec --provider-runner openai" in kit_readme, kit_readme assert "validate intake" in kit_readme, kit_readme assert "Artifact Checklist" in kit_readme, kit_readme assert "Source Evidence Snapshot" in kit_readme, kit_readme @@ -236,6 +241,8 @@ def main() -> None: assert "
Current
0
" in kit_html, kit_html assert "
Expected
>0
" in kit_html, kit_html assert "World-Class Evidence Submission Kit" in kit_html, kit_html + assert "Execution Runbook" in kit_html, kit_html + assert "output-exec --provider-runner openai" in kit_html, kit_html assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html native_kit_dir = TMP / "native_permission_kit" diff --git a/tests/verify_world_class_evidence_ledger.py b/tests/verify_world_class_evidence_ledger.py index ac67154..736572f 100644 --- a/tests/verify_world_class_evidence_ledger.py +++ b/tests/verify_world_class_evidence_ledger.py @@ -113,6 +113,7 @@ def main() -> None: "native-client-telemetry", }, entries assert entries["provider-holdout"]["observed_state"]["model_executed_count"] == 0, entries["provider-holdout"] + assert any("output-exec --provider-runner openai" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] provider_source = {item["field"]: item for item in entries["provider-holdout"]["source_checklist"]} assert provider_source["model_executed_count"]["status"] == "blocked", provider_source assert provider_source["timing_observed_count"]["status"] == "pass", provider_source @@ -136,6 +137,8 @@ def main() -> None: assert "overclaim guard active: `true`" in markdown, markdown assert "submitted entries: `0`" in markdown, markdown assert "source checks:" in markdown, markdown + assert "Source Runbook" in markdown, markdown + assert "output-exec --provider-runner openai" in markdown, markdown assert "Source Evidence Checks" in markdown, markdown assert "| Provider model run | `0` | `>0` | `blocked` |" in markdown, markdown assert "`provider-holdout`" in markdown, markdown diff --git a/tests/verify_world_class_operator_runbook.py b/tests/verify_world_class_operator_runbook.py index c516e43..34135e9 100644 --- a/tests/verify_world_class_operator_runbook.py +++ b/tests/verify_world_class_operator_runbook.py @@ -121,6 +121,7 @@ def main() -> None: assert provider["review_state"] == "awaiting-submission", provider assert provider["source_accepted"] is False, provider assert provider["blocked_source_check_count"] == 2, provider + assert any("output-exec --provider-runner openai" in step for step in provider["execution_runbook"]), provider assert "Run provider-backed output-exec with real credentials." in provider["next_source_actions"], provider assert "Provider execution should return non-estimated token usage." in provider["next_source_actions"], provider assert provider["commands"]["prepare_submission"].startswith("python3 scripts/yao.py world-class-submission-kit"), provider @@ -140,6 +141,8 @@ def main() -> None: assert "Valid intake means ready for submission review; ledger review still requires passing source evidence." in markdown, markdown assert "| Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner |" in markdown, markdown assert "| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Run provider-backed output-exec with real credentials." in markdown, markdown + assert "Source Runbook" in markdown, markdown + assert "output-exec --provider-runner openai" in markdown, markdown assert "### Next Source Actions" in markdown, markdown assert "- Provider execution should return non-estimated token usage." in markdown, markdown assert "Source Evidence Snapshot" in markdown, markdown @@ -154,6 +157,8 @@ def main() -> None: assert f"Blocked{summary['source_blocked_count']}" in html, html assert "
Blocked
2
" in html, html assert "Next Source Actions" in html, html + assert "Source Runbook" in html, html + assert "output-exec --provider-runner openai" in html, html assert "Source Evidence Snapshot" in html, html assert "model_executed_count" in html, html assert "model_executed_count: 0 / >0" in html, html