frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Expose preflight submission kit handoff

Browse Source
This commit is contained in:
yaojingang
2026-06-16 02:05:34 +08:00
parent c91d01bc1b
commit 713831ab3a
17 changed files with 283 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
registry/index.json
+1 -1
View File
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
}
]
}
registry/packages/yao-meta-skill.json
+3 -3
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee",
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050"
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d",
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903"
},
"compatibility": {
"openai": "pass",
@@ -48,7 +48,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050",
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.11,
"duration_ms": 28.53,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.32,
"duration_ms": 27.53,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.75,
"duration_ms": 27.52,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.01,
"duration_ms": 29.95,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.77,
"duration_ms": 30.22,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.46,
"duration_ms": 28.17,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.53,
"duration_ms": 26.13,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.46,
"duration_ms": 25.95,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.86,
"duration_ms": 25.55,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.67,
"duration_ms": 28.76,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 27.11 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 26.32 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 25.75 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.01 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 29.77 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.46 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.53 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.46 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.86 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.67 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 28.53 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.53 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.52 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.95 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 30.22 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.17 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.13 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.95 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.55 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.76 | 98 | 100.0 | pass |
## Next Fixes
reports/package_verification.json
+1 -1
View File
@@ -8,7 +8,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050",
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903",
"archive_entry_count": 623,
"failure_count": 0,
"warning_count": 0
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050`
- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903`
- Failures: `0`
- Warnings: `0`
reports/registry_audit.json
+4 -4
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee",
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050"
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d",
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903"
},
"compatibility": {
"openai": "pass",
@@ -53,7 +53,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050",
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee`
- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050`
- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d`
- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903`
- Install simulated: `True`
## Compatibility
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 200,
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `200`
- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee`
- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d`
## Failures
reports/upgrade_check.json
+2 -2
View File
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050"
"to": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
"to": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
}
]
},
reports/world_class_evidence_preflight.json
+66 -1
View File
@@ -131,6 +131,20 @@
"next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.",
"submission_path": "evidence/world_class/submissions/provider-holdout.json",
"template_path": "evidence/world_class/templates/provider-holdout.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"submission_kit": {
"prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions",
"output_dir": "evidence/world_class/submissions",
"draft_path": "evidence/world_class/submissions/provider-holdout.json",
"template_path": "evidence/world_class/templates/provider-holdout.intake.json",
"drafts_count_as_evidence": false
},
"runbook": [
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
@@ -230,6 +244,20 @@
"next_action": "Assign a real reviewer identity before claiming human adjudication.",
"submission_path": "evidence/world_class/submissions/human-adjudication.json",
"template_path": "evidence/world_class/templates/human-adjudication.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"submission_kit": {
"prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions",
"output_dir": "evidence/world_class/submissions",
"draft_path": "evidence/world_class/submissions/human-adjudication.json",
"template_path": "evidence/world_class/templates/human-adjudication.intake.json",
"drafts_count_as_evidence": false
},
"runbook": [
"python3 scripts/yao.py output-review-kit --write-template",
"Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.",
@@ -323,6 +351,20 @@
"next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.",
"submission_path": "evidence/world_class/submissions/native-permission-enforcement.json",
"template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"submission_kit": {
"prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions",
"output_dir": "evidence/world_class/submissions",
"draft_path": "evidence/world_class/submissions/native-permission-enforcement.json",
"template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json",
"drafts_count_as_evidence": false
},
"runbook": [
"Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.",
"Update the generated target adapter only when the guard is actually enforced by that target.",
@@ -416,6 +458,20 @@
"next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.",
"submission_path": "evidence/world_class/submissions/native-client-telemetry.json",
"template_path": "evidence/world_class/templates/native-client-telemetry.intake.json",
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"submission_kit": {
"prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions",
"output_dir": "evidence/world_class/submissions",
"draft_path": "evidence/world_class/submissions/native-client-telemetry.json",
"template_path": "evidence/world_class/templates/native-client-telemetry.intake.json",
"drafts_count_as_evidence": false
},
"runbook": [
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
"Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.",
@@ -718,7 +774,16 @@
],
"submissions": {
"directory": "evidence/world_class/submissions",
"preflight_counts_submission_as_completion": false
"commands": {
"prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions",
"validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions",
"submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions",
"refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard ."
},
"submission_kit_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions",
"preflight_counts_submission_as_completion": false,
"drafts_count_as_evidence": false
},
"source_reports": {
"ledger": "reports/world_class_evidence_ledger.json",
reports/world_class_evidence_preflight.md
+16
View File
@@ -14,6 +14,18 @@ Generated at: `2026-06-16`
This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger.
## Submission Kit Handoff
- submissions directory: `evidence/world_class/submissions`
- prepare drafts: `python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions`
- validate intake: `python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions`
- review queue: `python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`
- refresh ledger: `python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions`
- guard claims: `python3 scripts/yao.py world-class-claim-guard .`
- drafts count as evidence: `false`
Generate the submission kit after the real provider, human, native-permission, or native-client work exists. The generated JSON drafts remain `template_only: true` until an operator edits them with real aggregate artifact references and matching SHA-256 digests.
## Evidence Items
| Evidence | Status | Intake | Review | Next action |
@@ -28,6 +40,7 @@ This preflight report checks whether an operator can start collecting the remain
- status: `blocked`
- ledger: `pending`
- submission: `evidence/world_class/submissions/provider-holdout.json`
- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions`
### Prechecks
@@ -51,6 +64,7 @@ This preflight report checks whether an operator can start collecting the remain
- status: `ready-for-human-review`
- ledger: `pending`
- submission: `evidence/world_class/submissions/human-adjudication.json`
- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions`
### Prechecks
@@ -74,6 +88,7 @@ This preflight report checks whether an operator can start collecting the remain
- status: `blocked`
- ledger: `pending`
- submission: `evidence/world_class/submissions/native-permission-enforcement.json`
- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions`
### Prechecks
@@ -96,6 +111,7 @@ This preflight report checks whether an operator can start collecting the remain
- status: `blocked`
- ledger: `pending`
- submission: `evidence/world_class/submissions/native-client-telemetry.json`
- prepare draft: `python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions`
### Prechecks
scripts/render_evidence_consistency.py
+45
View File
@@ -455,6 +455,51 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["world_class_preflight"]],
detail="Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness.",
)
preflight_submissions = (
world_class_preflight.get("submissions", {})
if isinstance(world_class_preflight.get("submissions", {}), dict)
else {}
)
preflight_commands = (
preflight_submissions.get("commands", {})
if isinstance(preflight_submissions.get("commands", {}), dict)
else {}
)
default_submissions_dir = "evidence/world_class/submissions"
expected_preflight_handoff = {
"directory": default_submissions_dir,
"drafts_count_as_evidence": False,
"preflight_counts_submission_as_completion": False,
"prepare_submission": f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir}",
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {default_submissions_dir}",
"submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {default_submissions_dir}",
"refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {default_submissions_dir}",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard .",
}
actual_preflight_handoff = {
"directory": preflight_submissions.get("directory"),
"drafts_count_as_evidence": preflight_submissions.get("drafts_count_as_evidence"),
"preflight_counts_submission_as_completion": preflight_submissions.get(
"preflight_counts_submission_as_completion"
),
"prepare_submission": preflight_commands.get("prepare_submission"),
"validate_intake": preflight_commands.get("validate_intake"),
"submission_review": preflight_commands.get("submission_review"),
"refresh_ledger": preflight_commands.get("refresh_ledger"),
"guard_claim": preflight_commands.get("guard_claim"),
}
compare_values(
checks,
key="preflight-submission-kit-handoff",
label="Preflight exposes a safe submission-kit handoff",
expected=expected_preflight_handoff,
actual=actual_preflight_handoff,
paths=[REQUIRED_REPORTS["world_class_preflight"]],
detail=(
"Preflight must give operators the exact draft, intake, review, ledger, and claim-guard commands "
"without letting drafts or submissions count as accepted evidence."
),
)
public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False
compare_values(
scripts/render_world_class_preflight.py
+44
View File
@@ -2,6 +2,7 @@
import argparse
import json
import os
import shlex
from datetime import date
from pathlib import Path
from typing import Any
@@ -137,6 +138,24 @@ def env_present(name: str) -> bool:
return bool(os.environ.get(name))
def shell_path(path: Path, root: Path) -> str:
return shlex.quote(rel_path(path, root))
def build_submission_commands(skill_dir: Path, submissions_dir: Path, evidence_key: str | None = None) -> dict[str, str]:
output_dir = shell_path(submissions_dir, skill_dir)
prepare = f"python3 scripts/yao.py world-class-submission-kit . --output-dir {output_dir}"
if evidence_key:
prepare = f"python3 scripts/yao.py world-class-submission-kit . --evidence-key {evidence_key} --output-dir {output_dir}"
return {
"prepare_submission": prepare,
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {output_dir}",
"submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {output_dir}",
"refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {output_dir}",
"guard_claim": "python3 scripts/yao.py world-class-claim-guard .",
}
def build_precheck(skill_dir: Path, evidence_key: str, spec: dict[str, Any]) -> dict[str, Any]:
kind = str(spec.get("kind", ""))
required = spec.get("required") is True
@@ -236,6 +255,7 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path |
item_source_rows = review_item.get("source_checklist", entry.get("source_checklist", []))
item_source_rows = item_source_rows if isinstance(item_source_rows, list) else []
status = item_status(entry, prechecks, item_source_rows)
item_commands = build_submission_commands(skill_dir, submissions_dir, key)
item = {
"evidence_key": key,
"label": entry.get("label", key),
@@ -251,6 +271,14 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path |
"next_action": first_next_action(prechecks, item_source_rows),
"submission_path": intake_by_key.get(key, {}).get("submission_path", ""),
"template_path": intake_by_key.get(key, {}).get("template_path", ""),
"commands": item_commands,
"submission_kit": {
"prepare_command": item_commands["prepare_submission"],
"output_dir": rel_path(submissions_dir, skill_dir),
"draft_path": intake_by_key.get(key, {}).get("submission_path", ""),
"template_path": intake_by_key.get(key, {}).get("template_path", ""),
"drafts_count_as_evidence": False,
},
"runbook": entry.get("runbook", []),
}
items.append(item)
@@ -294,7 +322,10 @@ def build_preflight(skill_dir: Path, generated_at: str, submissions_dir: Path |
"source_checklist": source_rows,
"submissions": {
"directory": rel_path(submissions_dir, skill_dir),
"commands": build_submission_commands(skill_dir, submissions_dir),
"submission_kit_command": build_submission_commands(skill_dir, submissions_dir)["prepare_submission"],
"preflight_counts_submission_as_completion": False,
"drafts_count_as_evidence": False,
},
"source_reports": {
"ledger": "reports/world_class_evidence_ledger.json",
@@ -328,6 +359,18 @@ def render_markdown(report: dict[str, Any]) -> str:
"",
"This preflight report checks whether an operator can start collecting the remaining external or human evidence. It never accepts evidence, prints secret values, or changes the world-class ledger.",
"",
"## Submission Kit Handoff",
"",
f"- submissions directory: `{report['submissions']['directory']}`",
f"- prepare drafts: `{report['submissions']['commands']['prepare_submission']}`",
f"- validate intake: `{report['submissions']['commands']['validate_intake']}`",
f"- review queue: `{report['submissions']['commands']['submission_review']}`",
f"- refresh ledger: `{report['submissions']['commands']['refresh_ledger']}`",
f"- guard claims: `{report['submissions']['commands']['guard_claim']}`",
f"- drafts count as evidence: `{str(report['submissions']['drafts_count_as_evidence']).lower()}`",
"",
"Generate the submission kit after the real provider, human, native-permission, or native-client work exists. The generated JSON drafts remain `template_only: true` until an operator edits them with real aggregate artifact references and matching SHA-256 digests.",
"",
"## Evidence Items",
"",
"| Evidence | Status | Intake | Review | Next action |",
@@ -347,6 +390,7 @@ def render_markdown(report: dict[str, Any]) -> str:
f"- status: `{item['status']}`",
f"- ledger: `{item['ledger_status']}`",
f"- submission: `{item.get('submission_path') or 'missing'}`",
f"- prepare draft: `{item['commands']['prepare_submission']}`",
"",
"### Prechecks",
"",
tests/verify_evidence_consistency.py
+30
View File
@@ -151,6 +151,9 @@ def main() -> None:
assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"]
assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"]
assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"]
assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[
"preflight-submission-kit-handoff"
]
assert checks["review-studio-no-overclaim"]["status"] == "pass", checks["review-studio-no-overclaim"]
assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[
"claim-guard-package-runtime-surface"
@@ -248,6 +251,33 @@ def main() -> None:
preflight_drift_checks["preflight-world-class-boundary"]
)
preflight_handoff_drift_root = TMP / "preflight-handoff-drift-skill"
copy_reports(preflight_handoff_drift_root)
preflight_handoff_path = preflight_handoff_drift_root / "reports" / "world_class_evidence_preflight.json"
preflight_handoff = json.loads(preflight_handoff_path.read_text(encoding="utf-8"))
preflight_handoff["submissions"]["drafts_count_as_evidence"] = True
preflight_handoff["submissions"]["commands"].pop("prepare_submission", None)
preflight_handoff_path.write_text(json.dumps(preflight_handoff, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
preflight_handoff_drift_proc = run(
[
sys.executable,
str(SCRIPT),
str(preflight_handoff_drift_root),
"--output-json",
str(TMP / "preflight_handoff_drift.json"),
"--output-md",
str(TMP / "preflight_handoff_drift.md"),
"--generated-at",
"2026-06-15",
]
)
assert preflight_handoff_drift_proc.returncode == 2, preflight_handoff_drift_proc.stdout
preflight_handoff_drift_payload = json.loads(preflight_handoff_drift_proc.stdout)
preflight_handoff_drift_checks = {item["key"]: item for item in preflight_handoff_drift_payload["checks"]}
assert preflight_handoff_drift_checks["preflight-submission-kit-handoff"]["status"] == "fail", (
preflight_handoff_drift_checks["preflight-submission-kit-handoff"]
)
release_flow_drift_root = TMP / "release-flow-drift-skill"
copy_reports(release_flow_drift_root)
agents_path = release_flow_drift_root / "AGENTS.md"
tests/verify_world_class_preflight.py
+46
View File
@@ -79,9 +79,36 @@ def main() -> None:
assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary
assert summary["source_blocked_count"] >= 6, summary
assert payload["submissions"]["preflight_counts_submission_as_completion"] is False, payload
assert payload["submissions"]["drafts_count_as_evidence"] is False, payload
assert payload["submissions"]["submission_kit_command"] == (
"python3 scripts/yao.py world-class-submission-kit . "
"--output-dir evidence/world_class/submissions"
), payload["submissions"]
submission_commands = payload["submissions"]["commands"]
assert submission_commands["prepare_submission"] == (
"python3 scripts/yao.py world-class-submission-kit . "
"--output-dir evidence/world_class/submissions"
), submission_commands
assert submission_commands["validate_intake"] == (
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
), submission_commands
assert submission_commands["submission_review"] == (
"python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions"
), submission_commands
assert submission_commands["refresh_ledger"] == (
"python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions"
), submission_commands
assert submission_commands["guard_claim"] == "python3 scripts/yao.py world-class-claim-guard .", submission_commands
provider = by_key(payload["items"], "provider-holdout")
assert provider["status"] == "blocked", provider
assert provider["commands"]["prepare_submission"] == (
"python3 scripts/yao.py world-class-submission-kit . "
"--evidence-key provider-holdout --output-dir evidence/world_class/submissions"
), provider
assert provider["submission_kit"]["drafts_count_as_evidence"] is False, provider
assert provider["submission_kit"]["output_dir"] == "evidence/world_class/submissions", provider
assert provider["submission_kit"]["draft_path"] == "evidence/world_class/submissions/provider-holdout.json", provider
provider_checks = {item["key"]: item for item in provider["prechecks"]}
assert provider_checks["openai-api-key"]["status"] == "missing", provider_checks
assert provider_checks["openai-api-key"]["actual"] == "not-set", provider_checks
@@ -106,6 +133,10 @@ def main() -> None:
assert "ready to claim world-class: `false`" in markdown, markdown
assert "preflight counts as evidence: `false`" in markdown, markdown
assert "credential value exposed: `false`" in markdown, markdown
assert "Submission Kit Handoff" in markdown, markdown
assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in markdown, markdown
assert "world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions" in markdown, markdown
assert "drafts count as evidence: `false`" in markdown, markdown
assert "values are never printed" in markdown, markdown
env_json = TMP / "preflight_with_env.json"
@@ -126,6 +157,21 @@ def main() -> None:
assert env_payload["summary"]["credential_value_exposed"] is False, env_payload
assert env_payload["summary"]["ready_to_claim_world_class"] is False, env_payload
spaced_dir = TMP / "submission kit spaced"
spaced_proc = run_preflight(
None,
"--submissions-dir",
str(spaced_dir),
"--output-json",
str(TMP / "preflight_spaced.json"),
"--output-md",
str(TMP / "preflight_spaced.md"),
)
spaced_payload = json.loads(spaced_proc.stdout)
quoted_spaced = "'tests/tmp_world_class_preflight/submission kit spaced'"
assert quoted_spaced in spaced_payload["submissions"]["commands"]["prepare_submission"], spaced_payload["submissions"]
assert quoted_spaced in by_key(spaced_payload["items"], "provider-holdout")["commands"]["prepare_submission"], spaced_payload["items"]
cli_proc = run_cli(
"--output-json",
str(TMP / "cli_preflight.json"),