diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index 40c0ba17..476c858a 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -2,6 +2,7 @@ import argparse import html import json +import shlex import shutil from datetime import date from pathlib import Path @@ -22,6 +23,10 @@ def rel_path(path: Path, root: Path) -> str: return str(path.resolve()) +def shell_path(path: Path, root: Path) -> str: + return shlex.quote(rel_path(path, root)) + + def write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") @@ -338,8 +343,8 @@ def build_submission_kit( "files": files, "evidence_items": items, "commands": { - "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {rel_path(output_dir, skill_dir)}", - "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {rel_path(output_dir, skill_dir)}", + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {shell_path(output_dir, skill_dir)}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {shell_path(output_dir, skill_dir)}", "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", }, "safety": { diff --git a/scripts/render_world_class_evidence_intake.py b/scripts/render_world_class_evidence_intake.py index 09785966..f07d3fda 100644 --- a/scripts/render_world_class_evidence_intake.py +++ b/scripts/render_world_class_evidence_intake.py @@ -101,7 +101,10 @@ def build_operator_checklist( f"--evidence-key {shlex.quote(key)} --output-dir {shell_path(submission_path.parent, skill_dir)}" ), "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {submissions_dir_arg}", - "refresh_ledger": "python3 scripts/yao.py world-class-ledger .", + "submission_review": ( + f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {submissions_dir_arg}" + ), + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {submissions_dir_arg}", "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", }, "must_collect": { @@ -255,7 +258,7 @@ def render_operator_checklist(items: list[dict[str, Any]]) -> list[str]: lines.append(f"- submission: `{item['submission_path']}`") lines.extend(["", "#### Commands", ""]) commands = item.get("commands", {}) - for label in ["prepare_submission", "validate_intake", "refresh_ledger", "guard_claim"]: + for label in ["prepare_submission", "validate_intake", "submission_review", "refresh_ledger", "guard_claim"]: if commands.get(label): lines.append(f"- {label}: `{commands[label]}`") must_collect = item.get("must_collect", {}) diff --git a/scripts/render_world_class_operator_runbook.py b/scripts/render_world_class_operator_runbook.py index 335caf84..5fb3fe90 100644 --- a/scripts/render_world_class_operator_runbook.py +++ b/scripts/render_world_class_operator_runbook.py @@ -62,7 +62,7 @@ def build_runbook_item( "commands": { "prepare_submission": commands.get("prepare_submission", ""), "validate_intake": commands.get("validate_intake", ""), - "review_queue": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "review_queue": commands.get("submission_review", ""), "refresh_ledger": commands.get("refresh_ledger", "python3 scripts/yao.py world-class-ledger ."), "guard_claim": commands.get("guard_claim", "python3 scripts/yao.py world-class-claim-guard ."), }, diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index d5c3250a..97ac71e1 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -493,6 +493,8 @@ def main() -> None: assert provider_checklist["readiness"] == "awaiting-submission", provider_checklist assert provider_checklist["submission_path"] == "evidence/world_class/submissions/provider-holdout.json", provider_checklist assert provider_checklist["commands"]["validate_intake"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", provider_checklist + assert provider_checklist["commands"]["submission_review"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", provider_checklist + assert provider_checklist["commands"]["refresh_ledger"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", provider_checklist assert "provider-backed model run" in provider_checklist["must_collect"]["provenance_requirements"], provider_checklist assert full_payload["data"]["world_class_claim_guard"]["summary"]["decision"] == "claim-guard-pass-evidence-pending", full_payload["data"]["world_class_claim_guard"] assert full_payload["data"]["world_class_claim_guard"]["summary"]["violation_count"] == 0, full_payload["data"]["world_class_claim_guard"] diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index d4b42526..4780b04c 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -146,6 +146,8 @@ def main() -> None: "--evidence-key provider-holdout --output-dir evidence/world_class/submissions" ), checklist["provider-holdout"] assert checklist["provider-holdout"]["commands"]["validate_intake"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"] + assert checklist["provider-holdout"]["commands"]["submission_review"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"] + assert checklist["provider-holdout"]["commands"]["refresh_ledger"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"] assert "provider-backed model run" in checklist["provider-holdout"]["must_collect"]["provenance_requirements"], checklist["provider-holdout"] assert "reports/output_execution_runs.json summary.model_executed_count > 0" in checklist["provider-holdout"]["must_collect"]["success_checks"], checklist["provider-holdout"] assert checklist["provider-holdout"]["anti_overclaim"]["local_command_runner_counts_as_provider_model"] is False, checklist["provider-holdout"] @@ -158,7 +160,8 @@ def main() -> None: assert "0 existing / 0 sha256 verified / 0 required verified / 1 refs" in markdown, markdown assert "`evidence/world_class/submissions/provider-holdout.json`" in markdown, markdown assert "`python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions`" in markdown, markdown - assert "`python3 scripts/yao.py world-class-ledger .`" in markdown, markdown + assert "`python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`" in markdown, markdown + assert "`python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions`" in markdown, markdown assert "Templates and planned work do not count as accepted evidence." in markdown, markdown assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown @@ -196,6 +199,15 @@ def main() -> None: assert draft_intake["submissions"][0]["evidence_key"] == "provider-holdout", draft_intake["submissions"] assert all(item["evidence_key"] != "unknown" for item in draft_intake["submissions"]), draft_intake["submissions"] + spaced_kit_dir = TMP / "submission kit spaced" + spaced_kit_proc = run_kit("--output-dir", str(spaced_kit_dir), "--evidence-key", "provider-holdout") + spaced_kit_payload = json.loads(spaced_kit_proc.stdout) + quoted_spaced_dir = "'tests/tmp_world_class_evidence_intake/submission kit spaced'" + assert quoted_spaced_dir in spaced_kit_payload["commands"]["validate_intake"], spaced_kit_payload["commands"] + assert quoted_spaced_dir in spaced_kit_payload["commands"]["refresh_ledger"], spaced_kit_payload["commands"] + spaced_readme = (spaced_kit_dir / "README.md").read_text(encoding="utf-8") + assert quoted_spaced_dir in spaced_readme, spaced_readme + existing_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout") existing_payload = json.loads(existing_proc.stdout) assert existing_payload["summary"]["existing_count"] == 1, existing_payload["summary"] @@ -223,6 +235,8 @@ def main() -> None: assert "source evidence checks still do not pass" in valid_checklist["provider-holdout"]["blocking_reason"], valid_checklist["provider-holdout"] assert valid_checklist["provider-holdout"]["submission_path"].endswith("tests/tmp_world_class_evidence_intake/valid_submissions/provider-holdout.json"), valid_checklist["provider-holdout"] assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["validate_intake"], valid_checklist["provider-holdout"] + assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["submission_review"], valid_checklist["provider-holdout"] + assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["refresh_ledger"], valid_checklist["provider-holdout"] placeholder_dir = TMP / "placeholder_submissions" placeholder_dir.mkdir() diff --git a/tests/verify_world_class_operator_runbook.py b/tests/verify_world_class_operator_runbook.py index 63385c25..e1fd8a97 100644 --- a/tests/verify_world_class_operator_runbook.py +++ b/tests/verify_world_class_operator_runbook.py @@ -167,6 +167,9 @@ def main() -> None: assert submitted_provider["intake_readiness"] == "source-evidence-incomplete", submitted_provider assert submitted_provider["review_state"] == "source-evidence-incomplete", submitted_provider assert submitted_provider["source_accepted"] is False, submitted_provider + assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["validate_intake"], submitted_provider + assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["review_queue"], submitted_provider + assert "tests/tmp_world_class_operator_runbook/valid_submissions" in submitted_provider["commands"]["refresh_ledger"], submitted_provider print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))