From 713831ab3a682d2bd0aa49e2c20ce90ce0f45e57 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 02:05:34 +0800 Subject: [PATCH] Expose preflight submission kit handoff --- registry/index.json | 2 +- registry/packages/yao-meta-skill.json | 6 +- reports/output_execution_runs.json | 20 +++--- reports/output_execution_runs.md | 20 +++--- reports/package_verification.json | 2 +- reports/package_verification.md | 2 +- reports/registry_audit.json | 8 +-- reports/registry_audit.md | 4 +- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/upgrade_check.json | 4 +- reports/world_class_evidence_preflight.json | 67 ++++++++++++++++++++- reports/world_class_evidence_preflight.md | 16 +++++ scripts/render_evidence_consistency.py | 45 ++++++++++++++ scripts/render_world_class_preflight.py | 44 ++++++++++++++ tests/verify_evidence_consistency.py | 30 +++++++++ tests/verify_world_class_preflight.py | 46 ++++++++++++++ 17 files changed, 283 insertions(+), 37 deletions(-) diff --git a/registry/index.json b/registry/index.json index a8b4da6..31ea0b4 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" + "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 22a9d80..f407a86 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee", - "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050" + "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d", + "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050", + "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 51e94aa..64a9314 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.11, + "duration_ms": 28.53, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.32, + "duration_ms": 27.53, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.75, + "duration_ms": 27.52, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.01, + "duration_ms": 29.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.77, + "duration_ms": 30.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.46, + "duration_ms": 28.17, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.53, + "duration_ms": 26.13, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.46, + "duration_ms": 25.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.86, + "duration_ms": 25.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.67, + "duration_ms": 28.76, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 74397ba..5ae5b82 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 27.11 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 26.32 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 25.75 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.01 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 29.77 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.46 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.53 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.46 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.86 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.67 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 28.53 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 27.53 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 27.52 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.95 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 30.22 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.17 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.13 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.95 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.55 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.76 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 2263dd5..ccc7dea 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050", + "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903", "archive_entry_count": 623, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index afc59f2..63d8571 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050` +- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 7524321..41e1703 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee", - "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050" + "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d", + "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050", + "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" + "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 6064891..0333021 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee` -- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050` +- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d` +- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903` - Install simulated: `True` ## Compatibility diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 64c158e..97c0525 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 200, - "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" + "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 19173af..7ffb814 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `200` -- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee` +- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d` ## Failures diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 8066c29..0720254 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050" + "to": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" + "to": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" } ] }, diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index 1bdf553..c033fdb 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -131,6 +131,20 @@ "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "drafts_count_as_evidence": false + }, "runbook": [ "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", "python3 scripts/yao.py skill-os2-audit . --generated-at ", @@ -230,6 +244,20 @@ "next_action": "Assign a real reviewer identity before claiming human adjudication.", "submission_path": "evidence/world_class/submissions/human-adjudication.json", "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "drafts_count_as_evidence": false + }, "runbook": [ "python3 scripts/yao.py output-review-kit --write-template", "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", @@ -323,6 +351,20 @@ "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "drafts_count_as_evidence": false + }, "runbook": [ "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", "Update the generated target adapter only when the guard is actually enforced by that target.", @@ -416,6 +458,20 @@ "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "drafts_count_as_evidence": false + }, "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -718,7 +774,16 @@ ], "submissions": { "directory": "evidence/world_class/submissions", - "preflight_counts_submission_as_completion": false + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "preflight_counts_submission_as_completion": false, + "drafts_count_as_evidence": false }, "source_reports": { "ledger": "reports/world_class_evidence_ledger.json", diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index 18ee86a..f8f246a 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -14,6 +14,18 @@ Generated at: `2026-06-16` This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger. +## Submission Kit Handoff + +- submissions directory: `evidence/world_class/submissions` +- prepare drafts: `python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions` +- validate intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions` +- review queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions` +- refresh ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions` +- guard claims: `python3 scripts/yao.py world-class-claim-guard .` +- drafts count as evidence: `false` + +Generate the submission kit after the real provider, human, native-permission, or native-client work exists. The generated JSON drafts remain `template_only: true` until an operator edits them with real aggregate artifact references and matching SHA-256 digests. + ## Evidence Items | Evidence | Status | Intake | Review | Next action | @@ -28,6 +40,7 @@ This preflight report checks whether an operator can start collecting the remain - status: `blocked` - ledger: `pending` - submission: `evidence/world_class/submissions/provider-holdout.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions` ### Prechecks @@ -51,6 +64,7 @@ This preflight report checks whether an operator can start collecting the remain - status: `ready-for-human-review` - ledger: `pending` - submission: `evidence/world_class/submissions/human-adjudication.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions` ### Prechecks @@ -74,6 +88,7 @@ This preflight report checks whether an operator can start collecting the remain - status: `blocked` - ledger: `pending` - submission: `evidence/world_class/submissions/native-permission-enforcement.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions` ### Prechecks @@ -96,6 +111,7 @@ This preflight report checks whether an operator can start collecting the remain - status: `blocked` - ledger: `pending` - submission: `evidence/world_class/submissions/native-client-telemetry.json` +- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions` ### Prechecks diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index 0557904..ca6a893 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -455,6 +455,51 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["world_class_preflight"]], detail="Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness.", ) + preflight_submissions = ( + world_class_preflight.get("submissions", {}) + if isinstance(world_class_preflight.get("submissions", {}), dict) + else {} + ) + preflight_commands = ( + preflight_submissions.get("commands", {}) + if isinstance(preflight_submissions.get("commands", {}), dict) + else {} + ) + default_submissions_dir = "evidence/world_class/submissions" + expected_preflight_handoff = { + "directory": default_submissions_dir, + "drafts_count_as_evidence": False, + "preflight_counts_submission_as_completion": False, + "prepare_submission": f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir}", + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {default_submissions_dir}", + "submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {default_submissions_dir}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {default_submissions_dir}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + } + actual_preflight_handoff = { + "directory": preflight_submissions.get("directory"), + "drafts_count_as_evidence": preflight_submissions.get("drafts_count_as_evidence"), + "preflight_counts_submission_as_completion": preflight_submissions.get( + "preflight_counts_submission_as_completion" + ), + "prepare_submission": preflight_commands.get("prepare_submission"), + "validate_intake": preflight_commands.get("validate_intake"), + "submission_review": preflight_commands.get("submission_review"), + "refresh_ledger": preflight_commands.get("refresh_ledger"), + "guard_claim": preflight_commands.get("guard_claim"), + } + compare_values( + checks, + key="preflight-submission-kit-handoff", + label="Preflight exposes a safe submission-kit handoff", + expected=expected_preflight_handoff, + actual=actual_preflight_handoff, + paths=[REQUIRED_REPORTS["world_class_preflight"]], + detail=( + "Preflight must give operators the exact draft, intake, review, ledger, and claim-guard commands " + "without letting drafts or submissions count as accepted evidence." + ), + ) public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False compare_values( diff --git a/scripts/render_world_class_preflight.py b/scripts/render_world_class_preflight.py index 807852a..ea7f375 100644 --- a/scripts/render_world_class_preflight.py +++ b/scripts/render_world_class_preflight.py @@ -2,6 +2,7 @@ import argparse import json import os +import shlex from datetime import date from pathlib import Path from typing import Any @@ -137,6 +138,24 @@ def env_present(name: str) -> bool: return bool(os.environ.get(name)) +def shell_path(path: Path, root: Path) -> str: + return shlex.quote(rel_path(path, root)) + + +def build_submission_commands(skill_dir: Path, submissions_dir: Path, evidence_key: str | None = None) -> dict[str, str]: + output_dir = shell_path(submissions_dir, skill_dir) + prepare = f"python3 scripts/yao.py world-class-submission-kit . --output-dir {output_dir}" + if evidence_key: + prepare = f"python3 scripts/yao.py world-class-submission-kit . --evidence-key {evidence_key} --output-dir {output_dir}" + return { + "prepare_submission": prepare, + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {output_dir}", + "submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {output_dir}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {output_dir}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + } + + def build_precheck(skill_dir: Path, evidence_key: str, spec: dict[str, Any]) -> dict[str, Any]: kind = str(spec.get("kind", "")) required = spec.get("required") is True @@ -236,6 +255,7 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | item_source_rows = review_item.get("source_checklist", entry.get("source_checklist", [])) item_source_rows = item_source_rows if isinstance(item_source_rows, list) else [] status = item_status(entry, prechecks, item_source_rows) + item_commands = build_submission_commands(skill_dir, submissions_dir, key) item = { "evidence_key": key, "label": entry.get("label", key), @@ -251,6 +271,14 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | "next_action": first_next_action(prechecks, item_source_rows), "submission_path": intake_by_key.get(key, {}).get("submission_path", ""), "template_path": intake_by_key.get(key, {}).get("template_path", ""), + "commands": item_commands, + "submission_kit": { + "prepare_command": item_commands["prepare_submission"], + "output_dir": rel_path(submissions_dir, skill_dir), + "draft_path": intake_by_key.get(key, {}).get("submission_path", ""), + "template_path": intake_by_key.get(key, {}).get("template_path", ""), + "drafts_count_as_evidence": False, + }, "runbook": entry.get("runbook", []), } items.append(item) @@ -294,7 +322,10 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path | "source_checklist": source_rows, "submissions": { "directory": rel_path(submissions_dir, skill_dir), + "commands": build_submission_commands(skill_dir, submissions_dir), + "submission_kit_command": build_submission_commands(skill_dir, submissions_dir)["prepare_submission"], "preflight_counts_submission_as_completion": False, + "drafts_count_as_evidence": False, }, "source_reports": { "ledger": "reports/world_class_evidence_ledger.json", @@ -328,6 +359,18 @@ def render_markdown(report: dict[str, Any]) -> str: "", "This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger.", "", + "## Submission Kit Handoff", + "", + f"- submissions directory: `{report['submissions']['directory']}`", + f"- prepare drafts: `{report['submissions']['commands']['prepare_submission']}`", + f"- validate intake: `{report['submissions']['commands']['validate_intake']}`", + f"- review queue: `{report['submissions']['commands']['submission_review']}`", + f"- refresh ledger: `{report['submissions']['commands']['refresh_ledger']}`", + f"- guard claims: `{report['submissions']['commands']['guard_claim']}`", + f"- drafts count as evidence: `{str(report['submissions']['drafts_count_as_evidence']).lower()}`", + "", + "Generate the submission kit after the real provider, human, native-permission, or native-client work exists. The generated JSON drafts remain `template_only: true` until an operator edits them with real aggregate artifact references and matching SHA-256 digests.", + "", "## Evidence Items", "", "| Evidence | Status | Intake | Review | Next action |", @@ -347,6 +390,7 @@ def render_markdown(report: dict[str, Any]) -> str: f"- status: `{item['status']}`", f"- ledger: `{item['ledger_status']}`", f"- submission: `{item.get('submission_path') or 'missing'}`", + f"- prepare draft: `{item['commands']['prepare_submission']}`", "", "### Prechecks", "", diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index e68ef09..9f51861 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -151,6 +151,9 @@ def main() -> None: assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"] assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"] assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"] + assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[ + "preflight-submission-kit-handoff" + ] assert checks["review-studio-no-overclaim"]["status"] == "pass", checks["review-studio-no-overclaim"] assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[ "claim-guard-package-runtime-surface" @@ -248,6 +251,33 @@ def main() -> None: preflight_drift_checks["preflight-world-class-boundary"] ) + preflight_handoff_drift_root = TMP / "preflight-handoff-drift-skill" + copy_reports(preflight_handoff_drift_root) + preflight_handoff_path = preflight_handoff_drift_root / "reports" / "world_class_evidence_preflight.json" + preflight_handoff = json.loads(preflight_handoff_path.read_text(encoding="utf-8")) + preflight_handoff["submissions"]["drafts_count_as_evidence"] = True + preflight_handoff["submissions"]["commands"].pop("prepare_submission", None) + preflight_handoff_path.write_text(json.dumps(preflight_handoff, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + preflight_handoff_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(preflight_handoff_drift_root), + "--output-json", + str(TMP / "preflight_handoff_drift.json"), + "--output-md", + str(TMP / "preflight_handoff_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert preflight_handoff_drift_proc.returncode == 2, preflight_handoff_drift_proc.stdout + preflight_handoff_drift_payload = json.loads(preflight_handoff_drift_proc.stdout) + preflight_handoff_drift_checks = {item["key"]: item for item in preflight_handoff_drift_payload["checks"]} + assert preflight_handoff_drift_checks["preflight-submission-kit-handoff"]["status"] == "fail", ( + preflight_handoff_drift_checks["preflight-submission-kit-handoff"] + ) + release_flow_drift_root = TMP / "release-flow-drift-skill" copy_reports(release_flow_drift_root) agents_path = release_flow_drift_root / "AGENTS.md" diff --git a/tests/verify_world_class_preflight.py b/tests/verify_world_class_preflight.py index 287e864..1d53af8 100644 --- a/tests/verify_world_class_preflight.py +++ b/tests/verify_world_class_preflight.py @@ -79,9 +79,36 @@ def main() -> None: assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary assert summary["source_blocked_count"] >= 6, summary assert payload["submissions"]["preflight_counts_submission_as_completion"] is False, payload + assert payload["submissions"]["drafts_count_as_evidence"] is False, payload + assert payload["submissions"]["submission_kit_command"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--output-dir evidence/world_class/submissions" + ), payload["submissions"] + submission_commands = payload["submissions"]["commands"] + assert submission_commands["prepare_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--output-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["validate_intake"] == ( + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["submission_review"] == ( + "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["refresh_ledger"] == ( + "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" + ), submission_commands + assert submission_commands["guard_claim"] == "python3 scripts/yao.py world-class-claim-guard .", submission_commands provider = by_key(payload["items"], "provider-holdout") assert provider["status"] == "blocked", provider + assert provider["commands"]["prepare_submission"] == ( + "python3 scripts/yao.py world-class-submission-kit . " + "--evidence-key provider-holdout --output-dir evidence/world_class/submissions" + ), provider + assert provider["submission_kit"]["drafts_count_as_evidence"] is False, provider + assert provider["submission_kit"]["output_dir"] == "evidence/world_class/submissions", provider + assert provider["submission_kit"]["draft_path"] == "evidence/world_class/submissions/provider-holdout.json", provider provider_checks = {item["key"]: item for item in provider["prechecks"]} assert provider_checks["openai-api-key"]["status"] == "missing", provider_checks assert provider_checks["openai-api-key"]["actual"] == "not-set", provider_checks @@ -106,6 +133,10 @@ def main() -> None: assert "ready to claim world-class: `false`" in markdown, markdown assert "preflight counts as evidence: `false`" in markdown, markdown assert "credential value exposed: `false`" in markdown, markdown + assert "Submission Kit Handoff" in markdown, markdown + assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in markdown, markdown + assert "world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions" in markdown, markdown + assert "drafts count as evidence: `false`" in markdown, markdown assert "values are never printed" in markdown, markdown env_json = TMP / "preflight_with_env.json" @@ -126,6 +157,21 @@ def main() -> None: assert env_payload["summary"]["credential_value_exposed"] is False, env_payload assert env_payload["summary"]["ready_to_claim_world_class"] is False, env_payload + spaced_dir = TMP / "submission kit spaced" + spaced_proc = run_preflight( + None, + "--submissions-dir", + str(spaced_dir), + "--output-json", + str(TMP / "preflight_spaced.json"), + "--output-md", + str(TMP / "preflight_spaced.md"), + ) + spaced_payload = json.loads(spaced_proc.stdout) + quoted_spaced = "'tests/tmp_world_class_preflight/submission kit spaced'" + assert quoted_spaced in spaced_payload["submissions"]["commands"]["prepare_submission"], spaced_payload["submissions"] + assert quoted_spaced in by_key(spaced_payload["items"], "provider-holdout")["commands"]["prepare_submission"], spaced_payload["items"] + cli_proc = run_cli( "--output-json", str(TMP / "cli_preflight.json"),