From 5517bcf7a4ab72b344d38804c97be5dd6acc4514 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 07:49:02 +0800 Subject: [PATCH] Mirror artifact roles in Review Studio --- assets/review-studio.css | 29 + registry/index.json | 4 +- registry/packages/yao-meta-skill.json | 8 +- reports/architecture_maintainability.json | 40 +- reports/architecture_maintainability.md | 16 +- reports/benchmark_reproducibility.json | 66 +- reports/benchmark_reproducibility.md | 37 +- reports/compiled_targets.json | 2 +- reports/context_budget.json | 22 +- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 12 +- reports/evidence_consistency.json | 176 +- reports/evidence_consistency.md | 5 +- reports/install_simulation.json | 20 +- reports/install_simulation.md | 4 +- reports/output_execution_runs.json | 20 +- reports/output_execution_runs.md | 20 +- reports/package_verification.json | 4 +- reports/package_verification.md | 2 +- reports/python_compatibility.json | 10 +- reports/python_compatibility.md | 4 +- reports/registry_audit.json | 12 +- reports/registry_audit.md | 4 +- reports/review-studio.html | 61 +- reports/review-studio.json | 1446 +++++++++++++++-- reports/review-viewer.json | 52 +- reports/review_annotations.json | 2 +- reports/review_waivers.json | 4 +- reports/review_waivers.md | 2 +- reports/runtime_permission_probes.json | 10 +- reports/security_trust_report.json | 30 +- reports/security_trust_report.md | 11 +- reports/skill-interpretation.html | 2 +- reports/skill-interpretation.json | 52 +- reports/skill-os-2-review.md | 6 +- reports/skill-overview.html | 2 +- reports/skill-overview.json | 52 +- reports/skill_atlas.json | 9 +- reports/skill_os2_audit.json | 4 +- reports/skill_os2_audit.md | 4 +- reports/skill_os2_coverage.json | 20 +- reports/skill_os2_coverage.md | 16 +- reports/skillops/daily/2026-06-16.json | 17 +- reports/skillops/daily/2026-06-16.md | 5 +- reports/skillops/weekly/2026-W25.json | 17 +- reports/skillops/weekly/2026-W25.md | 5 +- reports/upgrade_check.json | 10 +- .../evidence_consistency_artifact_roles.py | 201 +++ scripts/render_evidence_consistency.py | 102 +- scripts/review_studio_action_evidence.py | 91 ++ scripts/review_studio_data.py | 3 + skill_atlas/catalog.json | 7 +- skill_atlas/stale_skills.json | 2 +- tests/verify_evidence_consistency.py | 8 + tests/verify_review_studio.py | 18 + 55 files changed, 2178 insertions(+), 612 deletions(-) create mode 100644 scripts/evidence_consistency_artifact_roles.py diff --git a/assets/review-studio.css b/assets/review-studio.css index 67e8850..fe3bad0 100644 --- a/assets/review-studio.css +++ b/assets/review-studio.css @@ -245,6 +245,35 @@ font-size: 13px; overflow-wrap: anywhere; } + .action-artifact-roles { + padding: 0; + margin: 0; + list-style: none; + display: grid; + gap: 8px; + } + .action-artifact-roles li { + border: 1px solid var(--line); + border-radius: 8px; + background: #fff; + padding: 10px; + display: grid; + gap: 4px; + min-width: 0; + } + .action-artifact-roles strong { + color: var(--ink); + font-weight: 500; + overflow-wrap: anywhere; + } + .action-artifact-roles span, + .action-artifact-roles code, + .action-artifact-roles small { + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; + white-space: normal; + } .action-evidence-item dl { display: grid; grid-template-columns: 76px minmax(0, 1fr); diff --git a/registry/index.json b/registry/index.json index 294cd57..b844628 100644 --- a/registry/index.json +++ b/registry/index.json @@ -1,6 +1,6 @@ { "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 2c7a592..8a963fc 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, "compatibility": { "openai": "pass", @@ -48,10 +48,10 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" } diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index a9d62dc..43a5a94 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -1,13 +1,13 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 190, - "script_file_count": 121, + "python_file_count": 191, + "script_file_count": 122, "test_file_count": 69, - "internal_module_count": 38, + "internal_module_count": 39, "cli_script_count": 85, "command_handler_count": 68, "entrypoint_command_handler_count": 18, @@ -29,13 +29,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 896, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -52,7 +45,7 @@ }, { "path": "tests/verify_review_studio.py", - "lines": 816, + "lines": 834, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -64,6 +57,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 808, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/skill_report_model.py", "lines": 800, @@ -115,13 +115,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 896, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -138,7 +131,7 @@ }, { "path": "tests/verify_review_studio.py", - "lines": 816, + "lines": 834, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -150,6 +143,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 808, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/skill_report_model.py", "lines": 800, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index ab0e042..e140746 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -1,14 +1,14 @@ # Architecture Maintainability -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary - decision: `pass` -- python files: `190` -- scripts: `121` +- python files: `191` +- scripts: `122` - tests: `69` -- internal modules: `38` +- internal modules: `39` - CLI scripts: `85` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` @@ -30,11 +30,11 @@ No file-size hotspots found. | File | Lines | Kind | Recommended next split | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | -| `scripts/render_evidence_consistency.py` | `896` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `tests/verify_world_class_evidence_intake.py` | `867` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | -| `tests/verify_review_studio.py` | `816` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +| `tests/verify_review_studio.py` | `834` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/render_world_class_preflight.py` | `814` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | +| `scripts/render_evidence_consistency.py` | `808` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_model.py` | `800` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/compile_skill.py` | `723` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | @@ -45,11 +45,11 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | -| `scripts/render_evidence_consistency.py` | `896` | `cli-script` | `pass` | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` | | `tests/verify_world_class_evidence_intake.py` | `867` | `test` | `pass` | -| `tests/verify_review_studio.py` | `816` | `test` | `pass` | +| `tests/verify_review_studio.py` | `834` | `test` | `pass` | | `scripts/render_world_class_preflight.py` | `814` | `cli-script` | `pass` | +| `scripts/render_evidence_consistency.py` | `808` | `cli-script` | `pass` | | `scripts/skill_report_model.py` | `800` | `internal-module` | `pass` | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` | | `scripts/compile_skill.py` | `723` | `cli-script` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 37d952c..b4aee5d 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,23 +3,36 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 55, + "sample": [ + " M assets/review-studio.css", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9", - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "evidence_bundle_sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -37,14 +50,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 55 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -53,10 +67,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "ready": false, + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9" + "sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -137,8 +151,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "f18cf6872d644c0abb40195aebd4ef653a4b2a42171156328125732961f3ac66" + "bytes": 7964, + "sha256": "3138b6a7f2febc20fd3469cd9818cdc46c258876df52cba95370fb63134c6e55" }, { "label": "blind_review", @@ -172,43 +186,43 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 117850, - "sha256": "75308c6d3e516580260b38949acd3f030d542eed4fe109834f81ae79c3bc180a" + "bytes": 118486, + "sha256": "5c186e046fd801c632fa68abad74659ac4b76c0f5f57effca75597d78b5d365e" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 25061, - "sha256": "364e9bbc8dd4d163114fb758eb75126d9005412b1d1a2c5f7c229df2b77435a6" + "bytes": 25199, + "sha256": "dbd04851a3a6a14273a2e62823ee6d1d181d2e91b88c8c34d2a1b5ca284bc35a" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "fb6fc7fab931893d29e231fb312ff2f5cfa83c56ea49a2a0710e84d33f239a7c" + "sha256": "4d9b738198c580bff963756df5bffe4ed558037a44dc06c729f549c5d747d373" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "975f915246f9e585a8f542a6386b42a958301946ebae472f8c4117adefc25a9d" + "sha256": "19a978e0c83e6a7c2ac0a3a28b2beefab1af615e8b2a1589d15ea584b82d46d8" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "0a127a000c337bf9d180b30ac7ff4fefd6a3392aa7c4c4b822f6b3727fbcfd50" + "bytes": 8758, + "sha256": "ba0b8b088b64b1b60715075da4502048c50b2b00d922c6cc7bd3456dca148b34" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14310, - "sha256": "de180d2b67b017d5654e10e34d58d56ab0e4861968e282f34777f3677522027e" + "sha256": "a4dfc4e3af1167e3d6ced9444c41bc6cd2133bb67942fedba94c046814bf7866" }, { "label": "world_class_evidence_plan", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index f82cd73..6c614eb 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,19 +1,19 @@ # Benchmark Reproducibility Generated at: `2026-06-16` -Commit: `89637b99a23dfba8e6feb5542829e3e38bdf6d0c` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9` +Commit: `a441abf61becd4fd4f1a0031a8ba18d6feac95ce` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `25` - missing artifacts: `0` -- source contract sha256: `4ee62717cf47` -- archive sha256: `8b255266b249` +- source contract sha256: `314e6eea327e` +- archive sha256: `8d8b3a3e8297` - output cases: `5` - disclosed failure cases: `3` - reproduction commands: `23` @@ -22,8 +22,8 @@ Evidence bundle SHA256: `9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd3 - world-class ready: `false` - world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `55` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | @@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9` +- sha256: `937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e` ## Methodology Sections @@ -72,17 +73,17 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `f18cf6872d64` | +| output_execution | `reports/output_execution_runs.json` | present | `3138b6a7f2fe` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `bb8c72a9291e` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `75308c6d3e51` | -| python_compatibility | `reports/python_compatibility.json` | present | `364e9bbc8dd4` | -| registry_audit | `reports/registry_audit.json` | present | `fb6fc7fab931` | -| package_verification | `reports/package_verification.json` | present | `975f915246f9` | -| install_simulation | `reports/install_simulation.json` | present | `0a127a000c33` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `de180d2b67b0` | +| trust_report | `reports/security_trust_report.json` | present | `5c186e046fd8` | +| python_compatibility | `reports/python_compatibility.json` | present | `dbd04851a3a6` | +| registry_audit | `reports/registry_audit.json` | present | `4d9b738198c5` | +| package_verification | `reports/package_verification.json` | present | `19a978e0c83e` | +| install_simulation | `reports/install_simulation.json` | present | `ba0b8b088b64` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `a4dfc4e3af11` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `38e95161404d` | | world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `d08019f31a78` | | world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `4a9119aba62b` | diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index eeb3e74..78d28a9 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, diff --git a/reports/context_budget.json b/reports/context_budget.json index b8883db..96d021b 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1041817, + "other_text_tokens": 1045217, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1042614, - "deferred_resource_tokens": 482305, + "estimated_total_text_tokens": 1046014, + "deferred_resource_tokens": 484698, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121 + "estimated_tokens": 424733, + "file_count": 122 }, { "path": "references", @@ -29,15 +29,15 @@ }, { "path": "assets", - "estimated_tokens": 9877, + "estimated_tokens": 10031, "file_count": 3 } ], "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121 + "estimated_tokens": 424733, + "file_count": 122 } ], "deferred_resource_governance": { @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121, + "estimated_tokens": 424733, + "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 628, + "relevant_file_count": 630, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index 413256f..6dda1b3 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 482305 | `governed` | scripts:422494 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 484698 | `governed` | scripts:424733 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 685ffce..8be7ae4 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-16", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 482305, + "deferred_resource_tokens": 484698, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121 + "estimated_tokens": 424733, + "file_count": 122 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121, + "estimated_tokens": 424733, + "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 6db8396..e4479ca 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -4,14 +4,14 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "check_count": 34, - "pass_count": 34, + "check_count": 35, + "pass_count": 35, "warn_count": 0, "fail_count": 0, "decision": "consistent" }, "status_counts": { - "pass": 34, + "pass": 35, "warn": 0, "fail": 0 }, @@ -189,12 +189,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 482305/120000; top deferred scripts 422494; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484698/120000; top deferred scripts 424733; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 482305/120000; top deferred scripts 422494; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484698/120000; top deferred scripts 424733; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -207,8 +207,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": true, - "actual": true, + "expected": false, + "actual": false, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -248,8 +248,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", - "actual": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "expected": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", + "actual": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -261,30 +261,30 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -394,8 +394,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", - "actual": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "expected": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", + "actual": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -407,30 +407,30 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -1512,7 +1512,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -1527,7 +1527,7 @@ "file_count": 229 } ], - "file_count": 418, + "file_count": 419, "folder_count": 4, "distribution": [ { @@ -1552,7 +1552,7 @@ }, { "label": "scripts", - "value": 121 + "value": 122 }, { "label": "evals", @@ -1600,7 +1600,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -1615,7 +1615,7 @@ "file_count": 229 } ], - "file_count": 418, + "file_count": 419, "folder_count": 4, "distribution": [ { @@ -1640,7 +1640,7 @@ }, { "label": "scripts", - "value": 121 + "value": 122 }, { "label": "evals", @@ -1912,6 +1912,108 @@ ], "detail": "Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence." }, + { + "key": "review-studio-preflight-artifact-role-handoff", + "label": "Review Studio mirrors preflight artifact roles", + "status": "pass", + "expected": { + "provider-holdout": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 1, + "submission_ref_ready_count": 1, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "human-adjudication": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 8, + "supporting_evidence_ready_count": 8, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-permission-enforcement": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 11, + "supporting_evidence_ready_count": 11, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-client-telemetry": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + } + }, + "actual": { + "provider-holdout": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 1, + "submission_ref_ready_count": 1, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "human-adjudication": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 8, + "supporting_evidence_ready_count": 8, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-permission-enforcement": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 11, + "supporting_evidence_ready_count": 11, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + }, + "native-client-telemetry": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "submission_ref_copy_to_artifact_refs": true, + "supporting_evidence_copy_to_artifact_refs": false + } + }, + "paths": [ + "reports/world_class_evidence_preflight.json", + "reports/review-studio.json" + ], + "detail": "The Review Studio world-class action card must carry the same submission-ref versus supporting-evidence contract as the preflight handoff." + }, { "key": "review-studio-no-overclaim", "label": "Review Studio does not overclaim pending world-class evidence", @@ -2302,10 +2404,10 @@ "score `91`", "`16` gates", "`3` warnings", - "`35` declared internal modules", - "`86 / 86` CLI help smoke checks passing across `121` scripts", - "`646` zip entries", - "archive with `646` entries", + "`36` declared internal modules", + "`86 / 86` CLI help smoke checks passing across `122` scripts", + "`647` zip entries", + "archive with `647` entries", "`12` installer permission checks enforced", "`0` permission failures", "`25` required artifacts", diff --git a/reports/evidence_consistency.md b/reports/evidence_consistency.md index 75677a8..88d9627 100644 --- a/reports/evidence_consistency.md +++ b/reports/evidence_consistency.md @@ -5,8 +5,8 @@ Generated at: `2026-06-16` ## Summary - decision: `consistent` -- checks: `34` -- pass: `34` +- checks: `35` +- pass: `35` - warn: `0` - fail: `0` @@ -46,6 +46,7 @@ This gate compares generated evidence reports against each other. It does not cr | Benchmark report mirrors world-class evidence boundary | `pass` | Benchmark reproducibility must not overstate public claim readiness. | `reports/world_class_evidence_ledger.json`, `reports/benchmark_reproducibility.json` | | Preflight mirrors ledger without accepting evidence | `pass` | Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_preflight.json` | | Preflight exposes a safe submission-kit handoff | `pass` | Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence. | `reports/world_class_evidence_preflight.json`, `reports/world_class_evidence_preflight.html` | +| Review Studio mirrors preflight artifact roles | `pass` | The Review Studio world-class action card must carry the same submission-ref versus supporting-evidence contract as the preflight handoff. | `reports/world_class_evidence_preflight.json`, `reports/review-studio.json` | | Review Studio does not overclaim pending world-class evidence | `pass` | When world-class evidence is pending, Review Studio must stay in a review or warning posture. | `reports/world_class_evidence_ledger.json`, `reports/review-studio.json` | | Claim guard covers package and runtime claim surfaces | `pass` | The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted. | `reports/world_class_claim_guard.json`, `manifest.json`, `agents/interface.yaml`, `dist/manifest.json`, `dist/targets/openai/adapter.json`, `evidence/world_class/README.md`, `security/permission_policy.json`, `reports/world_class_evidence_ledger.json` | | World-class evidence workflows cover every pending ledger entry | `pass` | Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion. | `reports/world_class_evidence_ledger.json`, `reports/world_class_evidence_plan.json`, `reports/world_class_evidence_intake.json`, `reports/world_class_submission_review.json`, `reports/world_class_operator_runbook.json`, `reports/review-studio.json` | diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 967d999..35db458 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -1,14 +1,14 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 646, + "archive_entry_count": 647, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -26,7 +26,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -272,9 +272,9 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } } diff --git a/reports/install_simulation.md b/reports/install_simulation.md index 6750e00..b0d16da 100644 --- a/reports/install_simulation.md +++ b/reports/install_simulation.md @@ -1,7 +1,7 @@ # Install Simulation - OK: `True` -- Package directory: `dist` +- Package directory: `tests/tmp_review_studio/dist` - Archive extracted: `True` - Entrypoint loaded: `True` - Manifest loaded: `True` @@ -16,7 +16,7 @@ | Check | Status | Detail | | --- | --- | --- | -| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip | +| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip | | `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | | `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill | | `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable | diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 469a5f6..dedc5ef 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.14, + "duration_ms": 32.7, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.01, + "duration_ms": 33.92, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.62, + "duration_ms": 47.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.94, + "duration_ms": 35.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.52, + "duration_ms": 33.41, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.91, + "duration_ms": 32.72, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.82, + "duration_ms": 33.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.45, + "duration_ms": 33.3, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.74, + "duration_ms": 32.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.03, + "duration_ms": 33.52, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 2abcf83..9d8c352 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 31.14 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 33.01 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 32.62 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 32.94 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 35.52 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 31.91 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 30.82 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 30.45 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 29.74 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 30.03 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 32.7 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 33.92 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 47.38 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 35.6 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 33.41 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 32.72 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 33.95 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 33.3 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 32.04 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 33.52 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 0c59b76..6f35317 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", - "archive_entry_count": 646, + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", + "archive_entry_count": 647, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 8f215fb..a9d09be 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6` +- Archive SHA256: `8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index da55a9c..cc2eb6e 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -1,11 +1,11 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "root": ".", "summary": { "target_python": "3.11", - "file_count": 193, + "file_count": 194, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -118,6 +118,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/evidence_consistency_release.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index dcd90a8..fa4cccc 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -1,12 +1,12 @@ # Python Compatibility -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary - decision: `pass` - target python: `3.11` -- files scanned: `193` +- files scanned: `194` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index f2ab195..29983b5 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, "compatibility": { "openai": "pass", @@ -53,16 +53,16 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 66cb7d8..c5352cd 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322` -- Archive SHA256: `8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6` +- Package SHA256: `314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156` +- Archive SHA256: `8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 94cd3ef..2e13e2b 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -252,6 +252,35 @@ font-size: 13px; overflow-wrap: anywhere; } + .action-artifact-roles { + padding: 0; + margin: 0; + list-style: none; + display: grid; + gap: 8px; + } + .action-artifact-roles li { + border: 1px solid var(--line); + border-radius: 8px; + background: #fff; + padding: 10px; + display: grid; + gap: 4px; + min-width: 0; + } + .action-artifact-roles strong { + color: var(--ink); + font-weight: 500; + overflow-wrap: anywhere; + } + .action-artifact-roles span, + .action-artifact-roles code, + .action-artifact-roles small { + color: var(--muted); + font-size: 12px; + overflow-wrap: anywhere; + white-space: normal; + } .action-evidence-item dl { display: grid; grid-template-columns: 76px minmax(0, 1fr); @@ -711,12 +740,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

121 scripts scanned; secrets found

Py Compat0

193 files scanned for Python 3.11

Arch Debt0

899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

646 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint20/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

122 scripts scanned; secrets found

Py Compat0

194 files scanned for Python 3.11

Arch Debt0

899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock false

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

647 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 482305/120000; top deferred scripts 422494; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 121 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 193 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

190 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 484698/120000; top deferred scripts 424733; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
@@ -726,7 +755,7 @@

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  2. python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  3. Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  2. python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  3. Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -742,7 +771,7 @@

证据路径

- +
@@ -778,17 +807,17 @@
-

上下文

initial load 990/1000; deferred 482305/120000; top deferred scripts 422494; resource governance governed; quality density 131.3

+

上下文

initial load 990/1000; deferred 484698/120000; top deferred scripts 424733; resource governance governed; quality density 131.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
121
网络脚本
3
Help 失败
0
包体哈希
4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322
+

信任报告

Secret
0
脚本数
122
网络脚本
3
Help 失败
0
包体哈希
314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

-

Python 兼容

目标 Python
3.11
文件数
193
问题数
0
语法错误
0
F-string 3.11
0
+

Python 兼容

目标 Python
3.11
文件数
194
问题数
0
语法错误
0
F-string 3.11
0

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线；本地更高版本允许的新语法不能绕过兼容门禁。

@@ -808,14 +837,14 @@
-

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

+

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
-

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
是
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
-

周度队列

决策
curator-review
Week Id
2026-W25
日报数
1
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
是
Writes Source Files
否
Auto Patch Enabled
否
+

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
否
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
+

周度队列

决策
curator-review
Week Id
2026-W25
日报数
1
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
否
Writes Source Files
否
Auto Patch Enabled
否
@@ -827,7 +856,7 @@

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0

批准候选

-
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 0; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
无
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .
+
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 0; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
无
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .
@@ -843,13 +872,13 @@
-

蓝图覆盖

项目数
21
模块数
8
建议 PR
13
通过数
21
Warn Count
0
缺失数
0
Extension Track Count
4
Extension Covered Count
4
Extension Partial Count
0
Extension Planned Count
0
Adaptive Extension Ready
是
本地蓝图
是
世界级
否
待补证据
4
+

蓝图覆盖

项目数
21
模块数
8
建议 PR
13
通过数
20
Warn Count
1
缺失数
0
Extension Track Count
4
Extension Covered Count
4
Extension Partial Count
0
Extension Planned Count
0
Adaptive Extension Ready
是
本地蓝图
否
世界级
否
待补证据
4

覆盖边界

蓝图覆盖只证明 2.0 模块、建议 PR、脚本、报告和测试在本地闭环；public world-class 仍以 world-class evidence ledger 的真人和外部证据为准。

-

公开声明

本地复现
是
发布锁
是
可公开声明
否
声明阻断
4
Provider 证据
否
人审完成
否
世界级就绪
否
-

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
+

公开声明

本地复现
是
发布锁
否
可公开声明
否
声明阻断
5
Provider 证据
否
人审完成
否
世界级就绪
否
+

声明阻断

  • 阻断release lock is not clean or commit is unavailable
  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (6/13 pass, 7 blocked)
@@ -869,12 +898,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
646
失败数
0
警告数
0
归档哈希
8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
647
失败数
0
警告数
0
归档哈希
8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0
diff --git a/reports/review-studio.json b/reports/review-studio.json index f62c1c8..a1fd596 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -43,7 +43,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 990/1000; deferred 482305/120000; top deferred scripts 422494; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484698/120000; top deferred scripts 424733; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 121 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -67,7 +67,7 @@ "key": "python-compat", "label": "Python 兼容", "status": "pass", - "detail": "Python 3.11; 193 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "detail": "Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", "evidence": "reports/python_compatibility.json", "link": "python_compatibility.md" }, @@ -75,7 +75,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "190 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint", + "detail": "191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md" }, @@ -107,7 +107,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", + "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false", "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", "link": "adoption_drift_report.md" }, @@ -516,6 +516,33 @@ "evidence/world_class/intake.schema.json", "evidence/world_class/templates/provider-holdout.intake.json" ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 1, + "submission_ref_total_count": 1, + "supporting_evidence_ready_count": 6, + "supporting_evidence_total_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 1, + "total_count": 1, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 6, + "total_count": 6, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, "privacy_contract": [ "Do not commit provider credentials or environment dumps.", "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." @@ -601,6 +628,33 @@ "reports/output_review_adjudication.json", "reports/output_review_adjudication.md" ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 2, + "submission_ref_total_count": 2, + "supporting_evidence_ready_count": 8, + "supporting_evidence_total_count": 8, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 2, + "total_count": 2, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 8, + "total_count": 8, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, "privacy_contract": [ "Reviewer decisions should not include raw user data or private customer detail.", "The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.", @@ -679,6 +733,33 @@ "reports/install_simulation.json", "reports/install_simulation.md" ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 2, + "submission_ref_total_count": 2, + "supporting_evidence_ready_count": 11, + "supporting_evidence_total_count": 11, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 2, + "total_count": 2, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 11, + "total_count": 11, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, "privacy_contract": [ "Do not mark native_enforcement true for metadata-only fallbacks.", "Keep residual risks visible for targets that still rely on operator enforcement." @@ -763,6 +844,33 @@ "scripts/telemetry_native_host.py", "evidence/world_class/intake.schema.json" ], + "artifact_role_contract": { + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_ready_count": 2, + "submission_ref_total_count": 2, + "supporting_evidence_ready_count": 6, + "supporting_evidence_total_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "ready_count": 2, + "total_count": 2, + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "ready_count": 6, + "total_count": 6, + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, "privacy_contract": [ "Telemetry must remain metadata-only and local-first.", "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." @@ -800,6 +908,8 @@ "world_class_evidence_plan": "reports/world_class_evidence_plan.md", "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "world_class_evidence_intake": "reports/world_class_evidence_intake.md", + "world_class_evidence_preflight": "reports/world_class_evidence_preflight.md", + "world_class_evidence_preflight_html": "reports/world_class_evidence_preflight.html", "world_class_submission_review": "reports/world_class_submission_review.md", "world_class_operator_runbook": "reports/world_class_operator_runbook.md", "world_class_operator_runbook_html": "reports/world_class_operator_runbook.html", @@ -1328,7 +1438,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -1343,7 +1453,7 @@ "file_count": 229 } ], - "file_count": 418, + "file_count": 419, "folder_count": 4, "distribution": [ { @@ -1368,7 +1478,7 @@ }, { "label": "scripts", - "value": 121 + "value": 122 }, { "label": "evals", @@ -1498,7 +1608,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -1812,13 +1922,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9", - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "evidence_bundle_sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1836,11 +1946,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 55 }, - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1924,9 +2034,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 212, - "script_count": 121, - "internal_module_count": 35, + "scanned_files": 213, + "script_count": 122, + "internal_module_count": 36, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1944,8 +2054,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 212, - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_hash_file_count": 213, + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" }, "skill_atlas": { "skill_count": 12, @@ -1983,8 +2093,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, "compatibility": { "openai": "pass", @@ -2015,12 +2125,12 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -2031,8 +2141,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", - "archive_entry_count": 646, + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", + "archive_entry_count": 647, "failure_count": 0, "warning_count": 0 }, @@ -2043,7 +2153,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 646, + "archive_entry_count": 647, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -2110,12 +2220,12 @@ { "field": "archive_sha256", "from": "", - "to": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "to": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "to": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, @@ -4488,7 +4598,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.14, + "duration_ms": 32.7, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4516,7 +4626,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.01, + "duration_ms": 33.92, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4539,7 +4649,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.62, + "duration_ms": 47.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4567,7 +4677,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.94, + "duration_ms": 35.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4590,7 +4700,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 35.52, + "duration_ms": 33.41, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4618,7 +4728,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.91, + "duration_ms": 32.72, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4641,7 +4751,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.82, + "duration_ms": 33.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4668,7 +4778,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.45, + "duration_ms": 33.3, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4691,7 +4801,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.74, + "duration_ms": 32.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4720,7 +4830,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.03, + "duration_ms": 33.52, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5446,23 +5556,36 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 55, + "sample": [ + " M assets/review-studio.css", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9", - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "evidence_bundle_sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -5480,14 +5603,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 55 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -5496,10 +5620,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "ready": false, + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -5507,7 +5631,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9" + "sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5580,8 +5704,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "f18cf6872d644c0abb40195aebd4ef653a4b2a42171156328125732961f3ac66" + "bytes": 7964, + "sha256": "3138b6a7f2febc20fd3469cd9818cdc46c258876df52cba95370fb63134c6e55" }, { "label": "blind_review", @@ -5615,43 +5739,43 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 117850, - "sha256": "75308c6d3e516580260b38949acd3f030d542eed4fe109834f81ae79c3bc180a" + "bytes": 118486, + "sha256": "5c186e046fd801c632fa68abad74659ac4b76c0f5f57effca75597d78b5d365e" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 25061, - "sha256": "364e9bbc8dd4d163114fb758eb75126d9005412b1d1a2c5f7c229df2b77435a6" + "bytes": 25199, + "sha256": "dbd04851a3a6a14273a2e62823ee6d1d181d2e91b88c8c34d2a1b5ca284bc35a" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "fb6fc7fab931893d29e231fb312ff2f5cfa83c56ea49a2a0710e84d33f239a7c" + "sha256": "4d9b738198c580bff963756df5bffe4ed558037a44dc06c729f549c5d747d373" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "975f915246f9e585a8f542a6386b42a958301946ebae472f8c4117adefc25a9d" + "sha256": "19a978e0c83e6a7c2ac0a3a28b2beefab1af615e8b2a1589d15ea584b82d46d8" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "0a127a000c337bf9d180b30ac7ff4fefd6a3392aa7c4c4b822f6b3727fbcfd50" + "bytes": 8758, + "sha256": "ba0b8b088b64b1b60715075da4502048c50b2b00d922c6cc7bd3456dca148b34" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14310, - "sha256": "de180d2b67b017d5654e10e34d58d56ab0e4861968e282f34777f3677522027e" + "sha256": "a4dfc4e3af1167e3d6ced9444c41bc6cd2133bb67942fedba94c046814bf7866" }, { "label": "world_class_evidence_plan", @@ -5860,22 +5984,22 @@ "item_count": 21, "module_count": 8, "recommended_pr_count": 13, - "pass_count": 21, - "warn_count": 0, + "pass_count": 20, + "warn_count": 1, "missing_count": 0, "extension_track_count": 4, "extension_partial_count": 0, "extension_planned_count": 0, "extension_covered_count": 4, "adaptive_extension_ready": true, - "local_blueprint_ready": true, + "local_blueprint_ready": false, "public_world_class_ready": false, "world_class_evidence_pending_count": 4, - "decision": "local-blueprint-covered-evidence-pending" + "decision": "continue-implementation" }, "status_counts": { - "pass": 21, - "warn": 0, + "pass": 20, + "warn": 1, "missing": 0 }, "extension_status_counts": { @@ -5983,7 +6107,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "121 scripts; secrets 0; help failures 0", + "current": "122 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -6057,7 +6181,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 646; install failures 0", + "current": "archive entries 647; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -6464,9 +6588,9 @@ "key": "evidence-consistency", "category": "recommended-pr", "label": "Evidence Consistency", - "status": "pass", + "status": "warn", "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", - "current": "34 consistency checks", + "current": "35 consistency checks", "command": "make ci-test", "test": "tests/verify_evidence_consistency.py", "evidence": [ @@ -6665,7 +6789,7 @@ "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, @@ -11479,7 +11603,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "dist", + "package_dir": "tests/tmp_review_studio/dist", "expected_capabilities": [ "file_write", "network", @@ -11519,7 +11643,7 @@ { "target": "openai", "status": "pass", - "adapter": "dist/targets/openai/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11616,7 +11740,7 @@ { "target": "claude", "status": "pass", - "adapter": "dist/targets/claude/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11698,7 +11822,7 @@ { "target": "generic", "status": "pass", - "adapter": "dist/targets/generic/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11780,7 +11904,7 @@ { "target": "vscode", "status": "pass", - "adapter": "dist/targets/vscode/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11870,9 +11994,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 212, - "script_count": 121, - "internal_module_count": 35, + "scanned_files": 213, + "script_count": 122, + "internal_module_count": 36, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -11890,8 +12014,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 212, - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_hash_file_count": 213, + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" }, "failures": [], "warnings": [], @@ -12084,6 +12208,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/evidence_consistency_release.py", "interface": "internal-module", @@ -13643,7 +13781,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 35, + "skipped_count": 36, "failed_scripts": [], "results": [ { @@ -14508,6 +14646,10 @@ } ], "skipped": [ + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "reason": "internal module" + }, { "path": "scripts/evidence_consistency_release.py", "reason": "internal module" @@ -14839,11 +14981,11 @@ "python_compatibility": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "root": ".", "summary": { "target_python": "3.11", - "file_count": 193, + "file_count": 194, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -14956,6 +15098,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/evidence_consistency_release.py", "ok": true, @@ -16028,13 +16176,13 @@ "architecture_maintainability": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 190, - "script_file_count": 121, + "python_file_count": 191, + "script_file_count": 122, "test_file_count": 69, - "internal_module_count": 38, + "internal_module_count": 39, "cli_script_count": 85, "command_handler_count": 68, "entrypoint_command_handler_count": 18, @@ -16056,13 +16204,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 896, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -16079,7 +16220,7 @@ }, { "path": "tests/verify_review_studio.py", - "lines": 816, + "lines": 834, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -16091,6 +16232,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 808, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/skill_report_model.py", "lines": 800, @@ -16142,13 +16290,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 896, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -16165,7 +16306,7 @@ }, { "path": "tests/verify_review_studio.py", - "lines": 816, + "lines": 834, "kind": "test", "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." @@ -16177,6 +16318,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 808, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/skill_report_model.py", "lines": 800, @@ -16221,16 +16369,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1041817, + "other_text_tokens": 1045217, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1042614, - "deferred_resource_tokens": 482305, + "estimated_total_text_tokens": 1046014, + "deferred_resource_tokens": 484698, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121 + "estimated_tokens": 424733, + "file_count": 122 }, { "path": "references", @@ -16244,15 +16392,15 @@ }, { "path": "assets", - "estimated_tokens": 9877, + "estimated_tokens": 10031, "file_count": 3 } ], "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121 + "estimated_tokens": 424733, + "file_count": 122 } ], "deferred_resource_governance": { @@ -16274,14 +16422,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 422494, - "file_count": 121, + "estimated_tokens": 424733, + "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 628, + "relevant_file_count": 630, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 @@ -17125,7 +17273,7 @@ }, "catalog": { "workspace_root": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skills": [ { "name": "yao-meta-skill", @@ -17158,6 +17306,7 @@ "scripts/cross_packager.py", "scripts/diff_eval.py", "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", "scripts/evidence_consistency_release.py", "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", @@ -17335,8 +17484,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, @@ -17999,7 +18148,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 77, + "age_days": 74, "allowed_days": 31, "actionable": false, "scope": "example" @@ -18264,8 +18413,8 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "writes_source_files": false, "auto_patch_enabled": false, "failure_count": 0, @@ -18281,8 +18430,8 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "writes_source_files": false, "auto_patch_enabled": false, "failure_count": 0 @@ -18730,8 +18879,8 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, - "evidence_consistency_ok": true + "release_lock_ready": false, + "evidence_consistency_ok": false }, "actions": [ { @@ -18743,6 +18892,11 @@ "key": "close-world-class-evidence", "priority": "high", "action": "Collect accepted external or human evidence for the pending world-class ledger entries." + }, + { + "key": "refresh-evidence-consistency", + "priority": "high", + "action": "Regenerate evidence-consistency before using this report for release decisions." } ], "failures": [], @@ -18779,8 +18933,8 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "public_world_class_ready": false, "world_class_pending_count": 4, "writes_source_files": false, @@ -18799,8 +18953,8 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "public_world_class_ready": false, "world_class_pending_count": 4, "writes_source_files": false, @@ -19051,8 +19205,8 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "public_world_class_ready": false, "world_class_pending_count": 4 }, @@ -19066,6 +19220,11 @@ "key": "close-world-class-evidence", "priority": "high", "action": "Collect accepted external or human evidence for pending world-class ledger entries before public claims." + }, + { + "key": "refresh-evidence-consistency", + "priority": "high", + "action": "Regenerate evidence consistency before using this curator report for release decisions." } ], "failures": [], @@ -19086,7 +19245,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "summary": { "waiver_count": 0, "active_count": 0, @@ -19136,7 +19295,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { @@ -19166,7 +19325,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "reports/review_annotations_input.json", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -20473,6 +20632,965 @@ "markdown": "reports/world_class_evidence_intake.md" } }, + "world_class_evidence_preflight": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-16", + "skill_dir": ".", + "summary": { + "evidence_item_count": 4, + "precheck_count": 14, + "precheck_pass_count": 9, + "precheck_missing_count": 1, + "precheck_optional_count": 1, + "precheck_human_required_count": 1, + "precheck_external_required_count": 2, + "collection_ready_count": 1, + "collection_blocked_count": 3, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, + "pending_count": 4, + "ready_to_claim_world_class": false, + "credential_value_exposed": false, + "preflight_counts_as_evidence": false, + "decision": "collection-preflight-blocked" + }, + "status_counts": { + "blocked": 3, + "ready-for-human-review": 1 + }, + "precheck_status_counts": { + "pass": 9, + "missing": 1, + "optional": 1, + "human-required": 1, + "external-required": 2 + }, + "items": [ + { + "evidence_key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "openai-api-key", + "label": "Provider credential", + "kind": "env", + "required": true, + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "secret_value_redacted": true, + "env": "OPENAI_API_KEY", + "status": "missing", + "actual": "not-set", + "default": "" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "gpt-4.1-mini" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "submission_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/provider-holdout.json", + "template_path": "evidence/world_class/templates/provider-holdout.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 1, + "submission_ref_ready_count": 1, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + }, + { + "evidence_key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "ready-for-human-review", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Import real A/B decisions with `python3 scripts/yao.py output-review-import --input --run-adjudication`.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + } + ], + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + } + ], + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "submission_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/human-adjudication.json", + "template_path": "evidence/world_class/templates/human-adjudication.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 8, + "supporting_evidence_ready_count": 8, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.", + "python3 scripts/yao.py output-review-import --input --run-adjudication", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "submission_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-permission-enforcement.json", + "template_path": "evidence/world_class/templates/native-permission-enforcement.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 11, + "supporting_evidence_ready_count": 11, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + }, + { + "evidence_key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "blocked", + "ledger_status": "pending", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", + "source_accepted": false, + "prechecks": [ + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit": { + "prepare_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions", + "prefill_command": "python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions --prefill-artifacts", + "output_dir": "evidence/world_class/submissions", + "draft_path": "evidence/world_class/submissions/native-client-telemetry.json", + "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 2, + "submission_ref_ready_count": 2, + "supporting_evidence_total_count": 6, + "supporting_evidence_ready_count": 6, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + }, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false + }, + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" + ] + } + ], + "prechecks": [ + { + "evidence_key": "provider-holdout", + "key": "output-cases", + "label": "Output eval cases", + "kind": "file", + "required": true, + "next_action": "Keep output holdout cases available before provider execution.", + "secret_value_redacted": false, + "path": "evals/output/cases.jsonl", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-runner", + "label": "Provider runner", + "kind": "file", + "required": true, + "next_action": "Use the provider runner instead of the local command runner for model-backed evidence.", + "secret_value_redacted": false, + "path": "scripts/provider_output_eval_runner.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "provider-holdout", + "key": "openai-api-key", + "label": "Provider credential", + "kind": "env", + "required": true, + "next_action": "Set OPENAI_API_KEY in the operator shell; never commit or print the value.", + "secret_value_redacted": true, + "env": "OPENAI_API_KEY", + "status": "missing", + "actual": "not-set", + "default": "" + }, + { + "evidence_key": "provider-holdout", + "key": "provider-model", + "label": "Provider model", + "kind": "env", + "required": false, + "next_action": "Optionally set YAO_OUTPUT_EVAL_MODEL; the runbook defaults to gpt-4.1-mini.", + "secret_value_redacted": false, + "env": "YAO_OUTPUT_EVAL_MODEL", + "status": "optional", + "actual": "not-set", + "default": "gpt-4.1-mini" + }, + { + "evidence_key": "human-adjudication", + "key": "review-kit", + "label": "Blind review kit", + "kind": "file", + "required": true, + "next_action": "Open the blind review kit and record real reviewer choices.", + "secret_value_redacted": false, + "path": "reports/output_review_kit.html", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-template", + "label": "Decision template", + "kind": "file", + "required": true, + "next_action": "Import real A/B decisions with `python3 scripts/yao.py output-review-import --input --run-adjudication`.", + "secret_value_redacted": false, + "path": "reports/output_review_decisions.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "decision-importer", + "label": "Decision importer", + "kind": "file", + "required": true, + "next_action": "Use the importer to reject raw content fields and normalize reviewer decisions before adjudication.", + "secret_value_redacted": false, + "path": "scripts/import_output_review_decisions.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "human-adjudication", + "key": "human-reviewer", + "label": "Human reviewer", + "kind": "human", + "required": true, + "next_action": "Assign a real reviewer identity before claiming human adjudication.", + "secret_value_redacted": false, + "status": "human-required", + "actual": "external-human-action" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-policy", + "label": "Permission policy", + "kind": "file", + "required": true, + "next_action": "Keep approved high-permission capabilities explicit.", + "secret_value_redacted": false, + "path": "security/permission_policy.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "permission-probes", + "label": "Runtime probes", + "kind": "file", + "required": true, + "next_action": "Refresh runtime permission probes after packaging changes.", + "secret_value_redacted": false, + "path": "reports/runtime_permission_probes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-permission-enforcement", + "key": "native-guard", + "label": "Native guard", + "kind": "external", + "required": true, + "next_action": "Attach a real target-client or external installer runtime guard; metadata fallback is not enough.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + }, + { + "evidence_key": "native-client-telemetry", + "key": "native-host", + "label": "Native telemetry host", + "kind": "file", + "required": true, + "next_action": "Use the native host to receive metadata-only client events.", + "secret_value_redacted": false, + "path": "scripts/telemetry_native_host.py", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "hook-recipes", + "label": "Hook recipes", + "kind": "file", + "required": true, + "next_action": "Refresh telemetry hook recipes before external client installation.", + "secret_value_redacted": false, + "path": "reports/telemetry_hook_recipes.json", + "status": "pass", + "actual": "present" + }, + { + "evidence_key": "native-client-telemetry", + "key": "external-client", + "label": "External client", + "kind": "external", + "required": true, + "next_action": "Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events.", + "secret_value_redacted": false, + "status": "external-required", + "actual": "external-integration-required" + } + ], + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + }, + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + }, + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "submissions": { + "directory": "evidence/world_class/submissions", + "commands": { + "prepare_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "prepare_prefilled_submission": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "validate_intake": "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", + "submission_review": "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", + "refresh_ledger": "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard ." + }, + "submission_kit_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions", + "submission_kit_prefill_command": "python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts", + "preflight_counts_submission_as_completion": false, + "drafts_count_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "artifact_role_contract": { + "schema_version": "1.0", + "role_source": "world-class-submission-kit", + "counts_as_evidence": false, + "artifact_prefill_counts_as_evidence": false, + "submission_ref_total_count": 7, + "submission_ref_ready_count": 7, + "supporting_evidence_total_count": 31, + "supporting_evidence_ready_count": 31, + "roles": [ + { + "role": "submission-ref", + "label": "Submission refs", + "copy_to_artifact_refs": true, + "description": "Rows marked submission-ref are the aggregate paths expected in artifact_refs." + }, + { + "role": "supporting-evidence", + "label": "Supporting evidence", + "copy_to_artifact_refs": false, + "description": "Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs." + } + ] + } + }, + "source_reports": { + "ledger": "reports/world_class_evidence_ledger.json", + "intake": "reports/world_class_evidence_intake.json", + "submission_review": "reports/world_class_submission_review.json", + "operator_runbook": "reports/world_class_operator_runbook.json" + }, + "artifacts": { + "json": "reports/world_class_evidence_preflight.json", + "markdown": "reports/world_class_evidence_preflight.md", + "html": "reports/world_class_evidence_preflight.html" + } + }, "world_class_submission_review": { "schema_version": "1.0", "ok": true, @@ -22154,8 +23272,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, "compatibility": { "openai": "pass", @@ -22186,16 +23304,16 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -22211,7 +23329,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, @@ -22234,8 +23352,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", - "archive_entry_count": 646, + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", + "archive_entry_count": 647, "failure_count": 0, "warning_count": 0 }, @@ -22893,14 +24011,14 @@ "install_simulation": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 646, + "archive_entry_count": 647, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -22918,7 +24036,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -23164,16 +24282,16 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } }, "upgrade_check": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -23242,12 +24360,12 @@ { "field": "archive_sha256", "from": "", - "to": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "to": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "to": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, @@ -23262,8 +24380,8 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } }, "manifest": { diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 25297a0..2077351 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -513,7 +513,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -528,7 +528,7 @@ "file_count": 229 } ], - "file_count": 418, + "file_count": 419, "folder_count": 4, "distribution": [ { @@ -553,7 +553,7 @@ }, { "label": "scripts", - "value": 121 + "value": 122 }, { "label": "evals", @@ -683,7 +683,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -997,13 +997,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9", - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "evidence_bundle_sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1021,11 +1021,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 55 }, - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1109,9 +1109,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 212, - "script_count": 121, - "internal_module_count": 35, + "scanned_files": 213, + "script_count": 122, + "internal_module_count": 36, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1129,8 +1129,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 212, - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_hash_file_count": 213, + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" }, "skill_atlas": { "skill_count": 12, @@ -1168,8 +1168,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, "compatibility": { "openai": "pass", @@ -1200,12 +1200,12 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1216,8 +1216,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", - "archive_entry_count": 646, + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", + "archive_entry_count": 647, "failure_count": 0, "warning_count": 0 }, @@ -1228,7 +1228,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 646, + "archive_entry_count": 647, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1295,12 +1295,12 @@ { "field": "archive_sha256", "from": "", - "to": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "to": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "to": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, diff --git a/reports/review_annotations.json b/reports/review_annotations.json index 76fadc1..66f2b1d 100644 --- a/reports/review_annotations.json +++ b/reports/review_annotations.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "reports/review_annotations_input.json", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, diff --git a/reports/review_waivers.json b/reports/review_waivers.json index aa2558e..07eeccb 100644 --- a/reports/review_waivers.json +++ b/reports/review_waivers.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "summary": { "waiver_count": 0, "active_count": 0, @@ -52,7 +52,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { diff --git a/reports/review_waivers.md b/reports/review_waivers.md index 6c8d594..dbc942d 100644 --- a/reports/review_waivers.md +++ b/reports/review_waivers.md @@ -35,7 +35,7 @@ - waiver allowed: `true` - risk: review pending 5; model-executed 0; output failures 0 - evidence: `reports/output_review_adjudication.md` -- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-16 --evidence reports/output_review_adjudication.md` +- verification: `python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md` - world-class boundary: Does not count as provider, human, or public world-class completion evidence. #### Required Review diff --git a/reports/runtime_permission_probes.json b/reports/runtime_permission_probes.json index 9c2ffd6..59c9a1a 100644 --- a/reports/runtime_permission_probes.json +++ b/reports/runtime_permission_probes.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "dist", + "package_dir": "tests/tmp_review_studio/dist", "expected_capabilities": [ "file_write", "network", @@ -42,7 +42,7 @@ { "target": "openai", "status": "pass", - "adapter": "dist/targets/openai/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -139,7 +139,7 @@ { "target": "claude", "status": "pass", - "adapter": "dist/targets/claude/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -221,7 +221,7 @@ { "target": "generic", "status": "pass", - "adapter": "dist/targets/generic/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -303,7 +303,7 @@ { "target": "vscode", "status": "pass", - "adapter": "dist/targets/vscode/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 368345d..2629da2 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 212, - "script_count": 121, - "internal_module_count": 35, + "scanned_files": 213, + "script_count": 122, + "internal_module_count": 36, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 212, - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_hash_file_count": 213, + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" }, "failures": [], "warnings": [], @@ -216,6 +216,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/evidence_consistency_release.py", "interface": "internal-module", @@ -1775,7 +1789,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 35, + "skipped_count": 36, "failed_scripts": [], "results": [ { @@ -2640,6 +2654,10 @@ } ], "skipped": [ + { + "path": "scripts/evidence_consistency_artifact_roles.py", + "reason": "internal module" + }, { "path": "scripts/evidence_consistency_release.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 503fa84..216e344 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `212` -- Scripts: `121` -- Internal script modules: `35` +- Scanned files: `213` +- Scripts: `122` +- Internal script modules: `36` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `212` -- Package SHA256: `4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322` +- Package hash files: `213` +- Package SHA256: `314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156` ## Failures @@ -75,6 +75,7 @@ | scripts/cross_packager.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/diff_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/emit_telemetry_event.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/evidence_consistency_artifact_roles.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts. | | scripts/evidence_consistency_release.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to verify release evidence refresh instructions. | | scripts/evidence_consistency_world_class.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks. | | scripts/export_skill_ir.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html index 8435d94..d04bb40 100644 --- a/reports/skill-interpretation.html +++ b/reports/skill-interpretation.html @@ -930,7 +930,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布Asset Distribution418项418 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+
资产分布Asset Distribution419项419 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index d4241a8..a73c6cc 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -513,7 +513,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -528,7 +528,7 @@ "file_count": 229 } ], - "file_count": 418, + "file_count": 419, "folder_count": 4, "distribution": [ { @@ -553,7 +553,7 @@ }, { "label": "scripts", - "value": 121 + "value": 122 }, { "label": "evals", @@ -687,7 +687,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -1001,13 +1001,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9", - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "evidence_bundle_sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1025,11 +1025,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 55 }, - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1113,9 +1113,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 212, - "script_count": 121, - "internal_module_count": 35, + "scanned_files": 213, + "script_count": 122, + "internal_module_count": 36, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1133,8 +1133,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 212, - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_hash_file_count": 213, + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" }, "skill_atlas": { "skill_count": 12, @@ -1172,8 +1172,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, "compatibility": { "openai": "pass", @@ -1204,12 +1204,12 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1220,8 +1220,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", - "archive_entry_count": 646, + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", + "archive_entry_count": 647, "failure_count": 0, "warning_count": 0 }, @@ -1232,7 +1232,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 646, + "archive_entry_count": 647, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1299,12 +1299,12 @@ { "field": "archive_sha256", "from": "", - "to": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "to": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "to": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index ec965e3..bfbe2b7 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -127,13 +127,13 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `35` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `121` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `36` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `122` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `646` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `646` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `647` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `647` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only review event, `1` adoption sample, adoption `100`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index e839fcb..f999d09 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -930,7 +930,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布Asset Distribution418项418 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+
资产分布Asset Distribution419项419 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index ed26943..4b9e23d 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -512,7 +512,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -527,7 +527,7 @@ "file_count": 229 } ], - "file_count": 418, + "file_count": 419, "folder_count": 4, "distribution": [ { @@ -552,7 +552,7 @@ }, { "label": "scripts", - "value": 121 + "value": 122 }, { "label": "evals", @@ -682,7 +682,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 121 + "file_count": 122 }, { "path": "evals", @@ -996,13 +996,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9c52f69857cc545bf9ce703780b1644753f6fdf1ab0b49f702c9dd31e9b46ab9", - "source_contract_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "evidence_bundle_sha256": "937f0bea34077800392fe7e8803f6af47b0b75d1bdc1d82419c98939d7e59f4e", + "source_contract_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 23, @@ -1020,11 +1020,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 55 }, - "commit": "89637b99a23dfba8e6feb5542829e3e38bdf6d0c", + "commit": "a441abf61becd4fd4f1a0031a8ba18d6feac95ce", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1108,9 +1108,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 212, - "script_count": 121, - "internal_module_count": 35, + "scanned_files": 213, + "script_count": 122, + "internal_module_count": 36, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1128,8 +1128,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 212, - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "package_hash_file_count": 213, + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" }, "skill_atlas": { "skill_count": 12, @@ -1167,8 +1167,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322", - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "package_sha256": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, "compatibility": { "openai": "pass", @@ -1199,12 +1199,12 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1215,8 +1215,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6", - "archive_entry_count": 646, + "archive_sha256": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0", + "archive_entry_count": 647, "failure_count": 0, "warning_count": 0 }, @@ -1227,7 +1227,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 646, + "archive_entry_count": 647, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1294,12 +1294,12 @@ { "field": "archive_sha256", "from": "", - "to": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "to": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "to": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 7ada18d..ae6eddd 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -44,7 +44,7 @@ }, "catalog": { "workspace_root": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skills": [ { "name": "yao-meta-skill", @@ -77,6 +77,7 @@ "scripts/cross_packager.py", "scripts/diff_eval.py", "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", "scripts/evidence_consistency_release.py", "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", @@ -254,8 +255,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, @@ -918,7 +919,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 77, + "age_days": 74, "allowed_days": 31, "actionable": false, "scope": "example" diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index f4bc065..882c3f8 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 121; help failures 0", + "current": "secrets 0; scripts 122; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 646; install failures 0; permission failures 0", + "current": "zip entries 647; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index d0db140..4bd96bd 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,11 +23,11 @@ Generated at: `2026-06-16` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 121; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 122; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 646; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 647; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 51521ec..b94ea14 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -7,22 +7,22 @@ "item_count": 21, "module_count": 8, "recommended_pr_count": 13, - "pass_count": 21, - "warn_count": 0, + "pass_count": 20, + "warn_count": 1, "missing_count": 0, "extension_track_count": 4, "extension_partial_count": 0, "extension_planned_count": 0, "extension_covered_count": 4, "adaptive_extension_ready": true, - "local_blueprint_ready": true, + "local_blueprint_ready": false, "public_world_class_ready": false, "world_class_evidence_pending_count": 4, - "decision": "local-blueprint-covered-evidence-pending" + "decision": "continue-implementation" }, "status_counts": { - "pass": 21, - "warn": 0, + "pass": 20, + "warn": 1, "missing": 0 }, "extension_status_counts": { @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "121 scripts; secrets 0; help failures 0", + "current": "122 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -204,7 +204,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 646; install failures 0", + "current": "archive entries 647; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -611,9 +611,9 @@ "key": "evidence-consistency", "category": "recommended-pr", "label": "Evidence Consistency", - "status": "pass", + "status": "warn", "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", - "current": "34 consistency checks", + "current": "35 consistency checks", "command": "make ci-test", "test": "tests/verify_evidence_consistency.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 8f4ff34..33bdfae 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -4,12 +4,12 @@ Generated at: `2026-06-16` ## Summary -- decision: `local-blueprint-covered-evidence-pending` -- local blueprint ready: `true` +- decision: `continue-implementation` +- local blueprint ready: `false` - public world-class ready: `false` -- pass: `21` / `21` +- pass: `20` / `21` - missing: `0` -- warn: `0` +- warn: `1` - reference extensions: `4` - extension covered: `4` - extension partial: `0` @@ -26,9 +26,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 121 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 122 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 646; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 647; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | @@ -48,7 +48,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Registry Package Format | `pass` | registry ok True | `make ci-test` | `tests/verify_registry_audit.py` | | Review Studio 2.0 | `pass` | 16 review gates | `make ci-test` | `tests/verify_review_studio.py` | | Migration V2 Docs | `pass` | migration guide present | `make ci-test` | `docs review` | -| Evidence Consistency | `pass` | 34 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | +| Evidence Consistency | `warn` | 35 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | ## Reference Extension Tracks @@ -213,7 +213,7 @@ These extension tracks come from the user-supplied 2.0 reference plan. They are ### Evidence Consistency - objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. -- status: `pass` +- status: `warn` - existing evidence: `scripts/render_evidence_consistency.py`, `reports/evidence_consistency.json`, `tests/verify_evidence_consistency.py` - next action: Keep this item covered as the implementation evolves. diff --git a/reports/skillops/daily/2026-06-16.json b/reports/skillops/daily/2026-06-16.json index a35cbc8..4ad68d9 100644 --- a/reports/skillops/daily/2026-06-16.json +++ b/reports/skillops/daily/2026-06-16.json @@ -14,8 +14,8 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "writes_source_files": false, "auto_patch_enabled": false, "failure_count": 0, @@ -31,8 +31,8 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "writes_source_files": false, "auto_patch_enabled": false, "failure_count": 0 @@ -480,8 +480,8 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, - "evidence_consistency_ok": true + "release_lock_ready": false, + "evidence_consistency_ok": false }, "actions": [ { @@ -493,6 +493,11 @@ "key": "close-world-class-evidence", "priority": "high", "action": "Collect accepted external or human evidence for the pending world-class ledger entries." + }, + { + "key": "refresh-evidence-consistency", + "priority": "high", + "action": "Regenerate evidence-consistency before using this report for release decisions." } ], "failures": [], diff --git a/reports/skillops/daily/2026-06-16.md b/reports/skillops/daily/2026-06-16.md index 105c41c..03950c3 100644 --- a/reports/skillops/daily/2026-06-16.md +++ b/reports/skillops/daily/2026-06-16.md @@ -14,8 +14,8 @@ Generated at: `2026-06-16` - local blueprint ready: `true` - public world-class ready: `false` - world-class pending: `4` -- release lock ready: `true` -- evidence consistency ok: `true` +- release lock ready: `false` +- evidence consistency ok: `false` This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence. @@ -35,6 +35,7 @@ This report is an operations cockpit for explicit-source SkillOps. It does not s - `high` Review proposal-only adaptation items before preparing any approval ledger entry. - `high` Collect accepted external or human evidence for the pending world-class ledger entries. +- `high` Regenerate evidence-consistency before using this report for release decisions. ## Opportunities diff --git a/reports/skillops/weekly/2026-W25.json b/reports/skillops/weekly/2026-W25.json index 998d23e..f88d378 100644 --- a/reports/skillops/weekly/2026-W25.json +++ b/reports/skillops/weekly/2026-W25.json @@ -15,8 +15,8 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "public_world_class_ready": false, "world_class_pending_count": 4, "writes_source_files": false, @@ -35,8 +35,8 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "public_world_class_ready": false, "world_class_pending_count": 4, "writes_source_files": false, @@ -287,8 +287,8 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, - "evidence_consistency_ok": true, + "release_lock_ready": false, + "evidence_consistency_ok": false, "public_world_class_ready": false, "world_class_pending_count": 4 }, @@ -302,6 +302,11 @@ "key": "close-world-class-evidence", "priority": "high", "action": "Collect accepted external or human evidence for pending world-class ledger entries before public claims." + }, + { + "key": "refresh-evidence-consistency", + "priority": "high", + "action": "Regenerate evidence consistency before using this curator report for release decisions." } ], "failures": [], diff --git a/reports/skillops/weekly/2026-W25.md b/reports/skillops/weekly/2026-W25.md index 4d75fc3..0d56ed2 100644 --- a/reports/skillops/weekly/2026-W25.md +++ b/reports/skillops/weekly/2026-W25.md @@ -13,8 +13,8 @@ Week: `2026-W25` - top_score: `88` - skill_count: `12` - actionable_portfolio_issue_count: `0` -- release_lock_ready: `True` -- evidence_consistency_ok: `True` +- release_lock_ready: `False` +- evidence_consistency_ok: `False` - public_world_class_ready: `False` - world_class_pending_count: `4` @@ -34,6 +34,7 @@ This report is a weekly curator cockpit for generated SkillOps evidence. It does - `high` Review the top ready-for-approval SkillOps opportunities before preparing any approval ledger entry. - `high` Collect accepted external or human evidence for pending world-class ledger entries before public claims. +- `high` Regenerate evidence consistency before using this curator report for release decisions. ## Curator Queue diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 07863e0..29bdd3b 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "8b255266b2499cfa87728094ca04744b2711f79ad80bd4137311e1087f8798c6" + "to": "8d8b3a3e8297052fec8122a058604cb682ed1af9d6cd6668711ffc6d198c37f0" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "4ee62717cf47c3cb8d92bde382eb227efe8fb3421e7b1271984217c4213bb322" + "to": "314e6eea327e8c956e53bd664a47f0951e784b52c6a342a57c4f576374569156" } ] }, @@ -90,7 +90,7 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } } diff --git a/scripts/evidence_consistency_artifact_roles.py b/scripts/evidence_consistency_artifact_roles.py new file mode 100644 index 0000000..12e82fe --- /dev/null +++ b/scripts/evidence_consistency_artifact_roles.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Compare artifact-role handoffs across world-class evidence reports.""" + +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to compare preflight and Review Studio artifact-role contracts." + + +def _compare_check( + *, + key: str, + label: str, + expected: Any, + actual: Any, + paths: list[str], + detail: str, +) -> dict[str, Any]: + return { + "key": key, + "label": label, + "status": "pass" if expected == actual else "fail", + "expected": expected, + "actual": actual, + "paths": paths, + "detail": detail, + } + + +def role_contract_signature(contract: dict[str, Any]) -> dict[str, Any]: + roles = { + str(item.get("role", "")): item + for item in contract.get("roles", []) + if isinstance(item, dict) and str(item.get("role", "")).strip() + } + return { + "role_source": contract.get("role_source"), + "counts_as_evidence": contract.get("counts_as_evidence"), + "artifact_prefill_counts_as_evidence": contract.get("artifact_prefill_counts_as_evidence"), + "submission_ref_total_count": contract.get("submission_ref_total_count"), + "submission_ref_ready_count": contract.get("submission_ref_ready_count"), + "supporting_evidence_total_count": contract.get("supporting_evidence_total_count"), + "supporting_evidence_ready_count": contract.get("supporting_evidence_ready_count"), + "submission_ref_copy_to_artifact_refs": roles.get("submission-ref", {}).get("copy_to_artifact_refs"), + "supporting_evidence_copy_to_artifact_refs": roles.get("supporting-evidence", {}).get( + "copy_to_artifact_refs" + ), + } + + +def preflight_role_signatures(world_class_preflight: dict[str, Any]) -> dict[str, dict[str, Any]]: + signatures: dict[str, dict[str, Any]] = {} + items = world_class_preflight.get("items", []) if isinstance(world_class_preflight, dict) else [] + for item in items: + if not isinstance(item, dict): + continue + key = str(item.get("evidence_key", "")).strip() + submission_kit = item.get("submission_kit", {}) if isinstance(item.get("submission_kit", {}), dict) else {} + contract = ( + submission_kit.get("artifact_role_contract", {}) + if isinstance(submission_kit.get("artifact_role_contract", {}), dict) + else {} + ) + if key and contract: + signatures[key] = role_contract_signature(contract) + return signatures + + +def review_studio_role_signatures(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]: + actions = review_studio.get("review_actions", []) if isinstance(review_studio, dict) else [] + for action in actions: + if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence": + continue + signatures: dict[str, dict[str, Any]] = {} + for step in action.get("evidence_steps", []): + if not isinstance(step, dict): + continue + key = str(step.get("key", "")).strip() + contract = ( + step.get("artifact_role_contract", {}) + if isinstance(step.get("artifact_role_contract", {}), dict) + else {} + ) + if key and contract: + signatures[key] = role_contract_signature(contract) + return signatures + return {} + + +def build_preflight_artifact_role_handoff_checks( + *, + skill_dir: Path, + world_class_preflight: dict[str, Any], + review_studio: dict[str, Any], + report_paths: dict[str, str], +) -> list[dict[str, Any]]: + preflight_submissions = ( + world_class_preflight.get("submissions", {}) + if isinstance(world_class_preflight.get("submissions", {}), dict) + else {} + ) + preflight_commands = ( + preflight_submissions.get("commands", {}) + if isinstance(preflight_submissions.get("commands", {}), dict) + else {} + ) + preflight_role_contract = ( + preflight_submissions.get("artifact_role_contract", {}) + if isinstance(preflight_submissions.get("artifact_role_contract", {}), dict) + else {} + ) + preflight_roles = { + str(item.get("role", "")): item for item in preflight_role_contract.get("roles", []) if isinstance(item, dict) + } + default_submissions_dir = "evidence/world_class/submissions" + expected_preflight_handoff = { + "directory": default_submissions_dir, + "drafts_count_as_evidence": False, + "preflight_counts_submission_as_completion": False, + "html_report": "reports/world_class_evidence_preflight.html", + "html_exists": True, + "prepare_submission": f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir}", + "prepare_prefilled_submission": ( + f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir} " + "--prefill-artifacts" + ), + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {default_submissions_dir}", + "submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {default_submissions_dir}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {default_submissions_dir}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + "artifact_prefill_counts_as_evidence": False, + "artifact_role_source": "world-class-submission-kit", + "artifact_role_counts_as_evidence": False, + "artifact_role_prefill_counts_as_evidence": False, + "submission_ref_role_present": True, + "supporting_evidence_role_present": True, + "submission_ref_copy_to_artifact_refs": True, + "supporting_evidence_copy_to_artifact_refs": False, + "submission_ref_total_present": True, + "supporting_evidence_total_present": True, + } + actual_preflight_handoff = { + "directory": preflight_submissions.get("directory"), + "drafts_count_as_evidence": preflight_submissions.get("drafts_count_as_evidence"), + "preflight_counts_submission_as_completion": preflight_submissions.get( + "preflight_counts_submission_as_completion" + ), + "html_report": world_class_preflight.get("artifacts", {}).get("html") + if isinstance(world_class_preflight.get("artifacts", {}), dict) + else None, + "html_exists": (skill_dir / "reports" / "world_class_evidence_preflight.html").exists(), + "prepare_submission": preflight_commands.get("prepare_submission"), + "prepare_prefilled_submission": preflight_commands.get("prepare_prefilled_submission"), + "validate_intake": preflight_commands.get("validate_intake"), + "submission_review": preflight_commands.get("submission_review"), + "refresh_ledger": preflight_commands.get("refresh_ledger"), + "guard_claim": preflight_commands.get("guard_claim"), + "artifact_prefill_counts_as_evidence": preflight_submissions.get("artifact_prefill_counts_as_evidence"), + "artifact_role_source": preflight_role_contract.get("role_source"), + "artifact_role_counts_as_evidence": preflight_role_contract.get("counts_as_evidence"), + "artifact_role_prefill_counts_as_evidence": preflight_role_contract.get( + "artifact_prefill_counts_as_evidence" + ), + "submission_ref_role_present": "submission-ref" in preflight_roles, + "supporting_evidence_role_present": "supporting-evidence" in preflight_roles, + "submission_ref_copy_to_artifact_refs": preflight_roles.get("submission-ref", {}).get( + "copy_to_artifact_refs" + ), + "supporting_evidence_copy_to_artifact_refs": preflight_roles.get("supporting-evidence", {}).get( + "copy_to_artifact_refs" + ), + "submission_ref_total_present": int(preflight_role_contract.get("submission_ref_total_count", 0)) > 0, + "supporting_evidence_total_present": int(preflight_role_contract.get("supporting_evidence_total_count", 0)) + > 0, + } + return [ + _compare_check( + key="preflight-submission-kit-handoff", + label="Preflight exposes a safe submission-kit handoff", + expected=expected_preflight_handoff, + actual=actual_preflight_handoff, + paths=[report_paths["world_class_preflight"], "reports/world_class_evidence_preflight.html"], + detail=( + "Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, " + "and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence." + ), + ), + _compare_check( + key="review-studio-preflight-artifact-role-handoff", + label="Review Studio mirrors preflight artifact roles", + expected=preflight_role_signatures(world_class_preflight), + actual=review_studio_role_signatures(review_studio), + paths=[report_paths["world_class_preflight"], report_paths["review_studio"]], + detail=( + "The Review Studio world-class action card must carry the same submission-ref versus " + "supporting-evidence contract as the preflight handoff." + ), + ), + ] diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index 5ef4f34..cc3d58f 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -6,6 +6,7 @@ from datetime import date from pathlib import Path from typing import Any +from evidence_consistency_artifact_roles import build_preflight_artifact_role_handoff_checks from evidence_consistency_release import build_release_evidence_flow_check from evidence_consistency_world_class import build_world_class_workflow_check from skill_ir_paths import find_skill_ir_path @@ -547,102 +548,13 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: paths=[REQUIRED_REPORTS["world_class_ledger"], REQUIRED_REPORTS["world_class_preflight"]], detail="Collection preflight may help operators gather evidence, but it must not print secrets or change world-class readiness.", ) - preflight_submissions = ( - world_class_preflight.get("submissions", {}) - if isinstance(world_class_preflight.get("submissions", {}), dict) - else {} - ) - preflight_commands = ( - preflight_submissions.get("commands", {}) - if isinstance(preflight_submissions.get("commands", {}), dict) - else {} - ) - preflight_role_contract = ( - preflight_submissions.get("artifact_role_contract", {}) - if isinstance(preflight_submissions.get("artifact_role_contract", {}), dict) - else {} - ) - preflight_roles = { - str(item.get("role", "")): item - for item in preflight_role_contract.get("roles", []) - if isinstance(item, dict) - } - default_submissions_dir = "evidence/world_class/submissions" - expected_preflight_handoff = { - "directory": default_submissions_dir, - "drafts_count_as_evidence": False, - "preflight_counts_submission_as_completion": False, - "html_report": "reports/world_class_evidence_preflight.html", - "html_exists": True, - "prepare_submission": f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir}", - "prepare_prefilled_submission": ( - f"python3 scripts/yao.py world-class-submission-kit . --output-dir {default_submissions_dir} " - "--prefill-artifacts" - ), - "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {default_submissions_dir}", - "submission_review": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {default_submissions_dir}", - "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {default_submissions_dir}", - "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", - "artifact_prefill_counts_as_evidence": False, - "artifact_role_source": "world-class-submission-kit", - "artifact_role_counts_as_evidence": False, - "artifact_role_prefill_counts_as_evidence": False, - "submission_ref_role_present": True, - "supporting_evidence_role_present": True, - "submission_ref_copy_to_artifact_refs": True, - "supporting_evidence_copy_to_artifact_refs": False, - "submission_ref_total_present": True, - "supporting_evidence_total_present": True, - } - actual_preflight_handoff = { - "directory": preflight_submissions.get("directory"), - "drafts_count_as_evidence": preflight_submissions.get("drafts_count_as_evidence"), - "preflight_counts_submission_as_completion": preflight_submissions.get( - "preflight_counts_submission_as_completion" - ), - "html_report": world_class_preflight.get("artifacts", {}).get("html") - if isinstance(world_class_preflight.get("artifacts", {}), dict) - else None, - "html_exists": (skill_dir / "reports" / "world_class_evidence_preflight.html").exists(), - "prepare_submission": preflight_commands.get("prepare_submission"), - "prepare_prefilled_submission": preflight_commands.get("prepare_prefilled_submission"), - "validate_intake": preflight_commands.get("validate_intake"), - "submission_review": preflight_commands.get("submission_review"), - "refresh_ledger": preflight_commands.get("refresh_ledger"), - "guard_claim": preflight_commands.get("guard_claim"), - "artifact_prefill_counts_as_evidence": preflight_submissions.get( - "artifact_prefill_counts_as_evidence" - ), - "artifact_role_source": preflight_role_contract.get("role_source"), - "artifact_role_counts_as_evidence": preflight_role_contract.get("counts_as_evidence"), - "artifact_role_prefill_counts_as_evidence": preflight_role_contract.get( - "artifact_prefill_counts_as_evidence" - ), - "submission_ref_role_present": "submission-ref" in preflight_roles, - "supporting_evidence_role_present": "supporting-evidence" in preflight_roles, - "submission_ref_copy_to_artifact_refs": preflight_roles.get("submission-ref", {}).get( - "copy_to_artifact_refs" - ), - "supporting_evidence_copy_to_artifact_refs": preflight_roles.get("supporting-evidence", {}).get( - "copy_to_artifact_refs" - ), - "submission_ref_total_present": int(preflight_role_contract.get("submission_ref_total_count", 0)) > 0, - "supporting_evidence_total_present": int( - preflight_role_contract.get("supporting_evidence_total_count", 0) + checks.extend( + build_preflight_artifact_role_handoff_checks( + skill_dir=skill_dir, + world_class_preflight=world_class_preflight, + review_studio=review_studio, + report_paths=REQUIRED_REPORTS, ) - > 0, - } - compare_values( - checks, - key="preflight-submission-kit-handoff", - label="Preflight exposes a safe submission-kit handoff", - expected=expected_preflight_handoff, - actual=actual_preflight_handoff, - paths=[REQUIRED_REPORTS["world_class_preflight"], "reports/world_class_evidence_preflight.html"], - detail=( - "Preflight must give operators the exact draft, SHA-prefill, intake, review, ledger, " - "and claim-guard commands without letting drafts, prefill, or submissions count as accepted evidence." - ), ) public_ready = bool(ledger_summary.get("ready_to_claim_world_class")) if isinstance(ledger_summary, dict) else False diff --git a/scripts/review_studio_action_evidence.py b/scripts/review_studio_action_evidence.py index 8047863..82cc14e 100644 --- a/scripts/review_studio_action_evidence.py +++ b/scripts/review_studio_action_evidence.py @@ -29,6 +29,50 @@ def first_text_items(*values: Any, limit: int = 4) -> list[str]: return [] +def artifact_role_rows(contract: dict[str, Any]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + roles = contract.get("roles", []) if isinstance(contract.get("roles", []), list) else [] + for role in roles: + if not isinstance(role, dict): + continue + role_name = str(role.get("role", "")).strip() + if role_name == "submission-ref": + ready_count = int(contract.get("submission_ref_ready_count", 0) or 0) + total_count = int(contract.get("submission_ref_total_count", 0) or 0) + elif role_name == "supporting-evidence": + ready_count = int(contract.get("supporting_evidence_ready_count", 0) or 0) + total_count = int(contract.get("supporting_evidence_total_count", 0) or 0) + else: + ready_count = 0 + total_count = 0 + rows.append( + { + "role": role_name, + "label": str(role.get("label", role_name)), + "ready_count": ready_count, + "total_count": total_count, + "copy_to_artifact_refs": role.get("copy_to_artifact_refs") is True, + "description": str(role.get("description", "")), + } + ) + return rows + + +def compact_artifact_role_contract(contract: dict[str, Any]) -> dict[str, Any]: + if not isinstance(contract, dict) or not contract: + return {} + return { + "role_source": str(contract.get("role_source", "")), + "counts_as_evidence": contract.get("counts_as_evidence") is True, + "artifact_prefill_counts_as_evidence": contract.get("artifact_prefill_counts_as_evidence") is True, + "submission_ref_ready_count": int(contract.get("submission_ref_ready_count", 0) or 0), + "submission_ref_total_count": int(contract.get("submission_ref_total_count", 0) or 0), + "supporting_evidence_ready_count": int(contract.get("supporting_evidence_ready_count", 0) or 0), + "supporting_evidence_total_count": int(contract.get("supporting_evidence_total_count", 0) or 0), + "roles": artifact_role_rows(contract), + } + + def world_class_action_steps(data: dict[str, Any]) -> list[dict[str, Any]]: ledger = data.get("world_class_evidence_ledger", {}) if isinstance(data, dict) else {} entries = ledger.get("entries", []) if isinstance(ledger, dict) else [] @@ -39,6 +83,13 @@ def world_class_action_steps(data: dict[str, Any]) -> list[dict[str, Any]]: for item in intake_items if isinstance(item, dict) and str(item.get("evidence_key", "")).strip() } + preflight = data.get("world_class_evidence_preflight", {}) if isinstance(data, dict) else {} + preflight_items = preflight.get("items", []) if isinstance(preflight, dict) else [] + preflight_by_key = { + str(item.get("evidence_key", "")): item + for item in preflight_items + if isinstance(item, dict) and str(item.get("evidence_key", "")).strip() + } steps = [] for entry in entries: if not isinstance(entry, dict): @@ -66,6 +117,12 @@ def world_class_action_steps(data: dict[str, Any]) -> list[dict[str, Any]]: if not runbook and isinstance(checklist.get("must_collect", {}), dict): runbook = checklist["must_collect"].get("runbook", []) must_collect = checklist.get("must_collect", {}) if isinstance(checklist.get("must_collect", {}), dict) else {} + preflight_item = preflight_by_key.get(key, {}) + submission_kit = ( + preflight_item.get("submission_kit", {}) + if isinstance(preflight_item.get("submission_kit", {}), dict) + else {} + ) steps.append( { "key": key, @@ -93,6 +150,11 @@ def world_class_action_steps(data: dict[str, Any]) -> list[dict[str, Any]]: entry.get("evidence_artifacts"), limit=5, ), + "artifact_role_contract": compact_artifact_role_contract( + submission_kit.get("artifact_role_contract", {}) + if isinstance(submission_kit.get("artifact_role_contract", {}), dict) + else {} + ), "privacy_contract": first_text_items(must_collect.get("privacy_contract"), entry.get("privacy_contract"), limit=3), } ) @@ -107,6 +169,32 @@ def render_small_list(items: list[Any], empty: str, ordered: bool = False) -> st return f"<{tag}>{rows}" +def render_artifact_role_contract(contract: dict[str, Any]) -> str: + roles = contract.get("roles", []) if isinstance(contract, dict) else [] + if not roles: + return "

暂无资产角色。

" + rows = [] + for role in roles: + rows.append( + "
  • " + f"{html.escape(str(role.get('role', '')))}" + f"{html.escape(str(role.get('ready_count', 0)))} / {html.escape(str(role.get('total_count', 0)))} ready" + f"artifact_refs: {html.escape(str(role.get('copy_to_artifact_refs') is True).lower())}" + f"{html.escape(str(role.get('description', '')))}" + "
    • " + ) + source = html.escape(str(contract.get("role_source", ""))) + counts = html.escape(str(contract.get("counts_as_evidence") is True).lower()) + prefill = html.escape(str(contract.get("artifact_prefill_counts_as_evidence") is True).lower()) + return ( + f"

    source: {source}; counts as evidence: {counts}; " + f"prefill counts as evidence: {prefill}

    " + "
      " + + "".join(rows) + + "
    " + ) + + def render_action_evidence_steps(steps: list[dict[str, Any]]) -> str: if not steps: return "" @@ -175,6 +263,9 @@ def render_action_evidence_steps(steps: list[dict[str, Any]]) -> str: "
    证据资产
    " + render_small_list(step.get("evidence_artifacts", []), "暂无证据资产。") + "
    " + "
    资产角色
    " + + render_artifact_role_contract(step.get("artifact_role_contract", {})) + + "
    " "
    隐私边界
    " + render_small_list(step.get("privacy_contract", []), "暂无隐私边界。") + "
    " diff --git a/scripts/review_studio_data.py b/scripts/review_studio_data.py index 571aac2..848455c 100644 --- a/scripts/review_studio_data.py +++ b/scripts/review_studio_data.py @@ -121,6 +121,8 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "world_class_evidence_plan": "reports/world_class_evidence_plan.md", "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "world_class_evidence_intake": "reports/world_class_evidence_intake.md", + "world_class_evidence_preflight": "reports/world_class_evidence_preflight.md", + "world_class_evidence_preflight_html": "reports/world_class_evidence_preflight.html", "world_class_submission_review": "reports/world_class_submission_review.md", "world_class_operator_runbook": "reports/world_class_operator_runbook.md", "world_class_operator_runbook_html": "reports/world_class_operator_runbook.html", @@ -176,6 +178,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: "adaptation_regression": load_json(reports / "adaptation_regression_report.json"), "world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"), "world_class_evidence_intake": load_json(reports / "world_class_evidence_intake.json"), + "world_class_evidence_preflight": load_json(reports / "world_class_evidence_preflight.json"), "world_class_submission_review": load_json(reports / "world_class_submission_review.json"), "world_class_operator_runbook": load_json(reports / "world_class_operator_runbook.json"), "world_class_claim_guard": load_json(reports / "world_class_claim_guard.json"), diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index f4a13ed..14b36f1 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -1,6 +1,6 @@ { "workspace_root": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skills": [ { "name": "yao-meta-skill", @@ -33,6 +33,7 @@ "scripts/cross_packager.py", "scripts/diff_eval.py", "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_artifact_roles.py", "scripts/evidence_consistency_release.py", "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", @@ -210,8 +211,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, diff --git a/skill_atlas/stale_skills.json b/skill_atlas/stale_skills.json index b57d361..6602e82 100644 --- a/skill_atlas/stale_skills.json +++ b/skill_atlas/stale_skills.json @@ -24,7 +24,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 77, + "age_days": 74, "allowed_days": 31, "actionable": false, "scope": "example" diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index 46ca2a4..c73e6d9 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -179,6 +179,14 @@ def main() -> None: assert preflight_handoff["supporting_evidence_role_present"] is True, preflight_handoff assert preflight_handoff["submission_ref_copy_to_artifact_refs"] is True, preflight_handoff assert preflight_handoff["supporting_evidence_copy_to_artifact_refs"] is False, preflight_handoff + assert checks["review-studio-preflight-artifact-role-handoff"]["status"] == "pass", checks[ + "review-studio-preflight-artifact-role-handoff" + ] + role_handoff = checks["review-studio-preflight-artifact-role-handoff"]["actual"] + assert role_handoff["provider-holdout"]["role_source"] == "world-class-submission-kit", role_handoff + assert role_handoff["provider-holdout"]["submission_ref_total_count"] == 1, role_handoff + assert role_handoff["provider-holdout"]["submission_ref_copy_to_artifact_refs"] is True, role_handoff + assert role_handoff["provider-holdout"]["supporting_evidence_copy_to_artifact_refs"] is False, role_handoff assert checks["review-studio-no-overclaim"]["status"] == "pass", checks["review-studio-no-overclaim"] assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[ "claim-guard-package-runtime-surface" diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index c026131..357aa1b 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -259,6 +259,7 @@ def main() -> None: "render_world_class_evidence_plan.py", "render_world_class_evidence_ledger.py", "render_world_class_evidence_intake.py", + "render_world_class_preflight.py", "render_world_class_submission_review.py", "render_world_class_operator_runbook.py", "render_world_class_claim_guard.py", @@ -424,6 +425,10 @@ def main() -> None: assert full_payload["evidence_paths"]["world_class_evidence_ledger"] == "reports/world_class_evidence_ledger.md", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_evidence_intake.md").exists(): assert full_payload["evidence_paths"]["world_class_evidence_intake"] == "reports/world_class_evidence_intake.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_preflight.md").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_preflight"] == "reports/world_class_evidence_preflight.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_preflight.html").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_preflight_html"] == "reports/world_class_evidence_preflight.html", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_submission_review.md").exists(): assert full_payload["evidence_paths"]["world_class_submission_review"] == "reports/world_class_submission_review.md", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_operator_runbook.md").exists(): @@ -543,6 +548,15 @@ def main() -> None: assert "provider-backed model run" in provider_action_step["provenance_requirements"], provider_action_step assert "reports/output_execution_runs.json summary.model_executed_count > 0" in provider_action_step["success_checks"], provider_action_step assert "reports/output_execution_runs.json" in provider_action_step["evidence_artifacts"], provider_action_step + provider_role_contract = provider_action_step["artifact_role_contract"] + assert provider_role_contract["role_source"] == "world-class-submission-kit", provider_role_contract + assert provider_role_contract["counts_as_evidence"] is False, provider_role_contract + assert provider_role_contract["artifact_prefill_counts_as_evidence"] is False, provider_role_contract + assert provider_role_contract["submission_ref_total_count"] == 1, provider_role_contract + assert provider_role_contract["submission_ref_ready_count"] == 1, provider_role_contract + provider_role_rows = {item["role"]: item for item in provider_role_contract["roles"]} + assert provider_role_rows["submission-ref"]["copy_to_artifact_refs"] is True, provider_role_rows + assert provider_role_rows["supporting-evidence"]["copy_to_artifact_refs"] is False, provider_role_rows assert any("provider credentials" in item for item in provider_action_step["privacy_contract"]), provider_action_step assert full_payload["data"]["world_class_evidence_ledger"]["summary"]["pending_count"] == 4, full_payload["data"]["world_class_evidence_ledger"] assert full_payload["data"]["world_class_evidence_intake"]["summary"]["decision"] == "awaiting-submissions", full_payload["data"]["world_class_evidence_intake"] @@ -689,6 +703,10 @@ def main() -> None: assert "native-client-telemetry.json" in html, html assert "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" in html, html assert "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions" in html, html + assert "资产角色" in html, html + assert "submission-ref" in html, html + assert "supporting-evidence" in html, html + assert "artifact_refs: true" in html, html assert "intake 只校验证据包格式、来源、隐私和反过度声明" in html, html assert "reports/world_class_evidence_intake.md" in html, html assert "reports/world_class_operator_runbook.html" in html, html
    路径Path作用Role类型Type
    SKILL.mdSkill 入口文件Skill entrypoint文件file
    README.md人类可读使用说明Human-readable usage guide文件file
    agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
    manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
    references扩展指导与复用资料Extended guidance and reusable notes目录folder
    scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
    evals触发与质量检查Trigger and quality checks目录folder
    reports生成的证据与总结报告Generated evidence and overview artifacts目录folder