diff --git a/registry/index.json b/registry/index.json index f7474981..40794b4b 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 460e6c7d..6ba2870f 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 8e610e48..8bebba9e 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-15T16:57:40Z", + "generated_at": "2026-06-15T17:08:05Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index f5cd3fb9..d1d9c529 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 173, - "script_file_count": 110, + "python_file_count": 174, + "script_file_count": 111, "test_file_count": 63, - "internal_module_count": 31, + "internal_module_count": 32, "cli_script_count": 81, "command_handler_count": 64, "entrypoint_command_handler_count": 18, @@ -16,7 +16,7 @@ "watch_line_threshold": 720, "block_line_threshold": 1500, "largest_file_lines": 899, - "watchlist_count": 9, + "watchlist_count": 8, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" @@ -29,13 +29,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 859, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 810, @@ -105,6 +98,13 @@ "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_review_viewer.py", + "lines": 685, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Split viewer data assembly from HTML section rendering." } ], "watchlist": [ @@ -115,13 +115,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 859, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 810, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 19e51a1b..45f3de88 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,17 +5,17 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `173` -- scripts: `110` +- python files: `174` +- scripts: `111` - tests: `63` -- internal modules: `31` +- internal modules: `32` - CLI scripts: `81` - Yao CLI command handlers: `64` - entrypoint command handlers: `18` - command modules: `6` - largest file lines: `899` - watch threshold lines: `720` -- watchlist: `9` +- watchlist: `8` - hotspots: `0` - blockers: `0` @@ -30,7 +30,6 @@ No file-size hotspots found. | File | Lines | Kind | Recommended next split | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | -| `scripts/render_evidence_consistency.py` | `859` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/yao_cli_parser.py` | `810` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_layout.py` | `808` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_model.py` | `801` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | @@ -44,7 +43,6 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | -| `scripts/render_evidence_consistency.py` | `859` | `cli-script` | `pass` | | `scripts/yao_cli_parser.py` | `810` | `internal-module` | `pass` | | `scripts/skill_report_layout.py` | `808` | `internal-module` | `pass` | | `scripts/skill_report_model.py` | `801` | `internal-module` | `pass` | @@ -55,6 +53,7 @@ No file-size hotspots found. | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | | `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` | | `scripts/review_studio_layout.py` | `694` | `internal-module` | `pass` | +| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | ## Release Rule diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 19e2d321..23f973de 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,23 +3,36 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 36, + "sample": [ + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json", + " M reports/install_simulation.json" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "evidence_bundle_sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -37,14 +50,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 36 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -53,10 +67,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "ready": false, + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe" + "sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -137,8 +151,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7964, - "sha256": "30b43454ef2a4802b13d554100789b1c9c7a1baec9bebcd6fc8cf0ea9830ba41" + "bytes": 7967, + "sha256": "56430f01ba194dc6dc793eb505f92fec7a8c46e1be4b71976a3d641c9b98c7ec" }, { "label": "blind_review", @@ -172,36 +186,36 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 109099, - "sha256": "491354ae881f6682687a62a98c9a4988bbeef931b24565fdf43e6f3a206fd449" + "bytes": 109726, + "sha256": "a3a6d4e6809ea0b78b8012e37ad68903a54876a06100434efad38ea6bfe9b2bd" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 22887, - "sha256": "be0b8550a4404d85a044b3ae612b39d94e8b493723e7aaf02386cace291b8262" + "bytes": 23022, + "sha256": "e53158aa045e19f51cbf61f4cb8b299c9ac2babeaacd5441ecb911eb99d8089d" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "870c455a5cd209ff8d9e8793861da074e668083d265230d04e38e8ee0ff1323a" + "sha256": "d6b523cc5cbb2d69c35989913e7e68dd4dc2bb04d3bcff88bbcb56fbe8be5084" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "c4acb824482d992e0e4339b7eeeb8d23d49da12d3414606222f4916c3ed09df6" + "sha256": "ae8ac3872be3299b3d2dc18a29128346a3ecbb579987b1b08156e305237b5119" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, "bytes": 8604, - "sha256": "28ceb014e202269bb96628391d056234794b3570ff25de0ef9bdfd7f755fd85f" + "sha256": "3e27a6f7678efe9d94528f811a9d3acf20cf41d6328ad5b5fa0be5714c6396a0" }, { "label": "skill_os2_audit", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 04455029..3d0b7310 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,19 +1,19 @@ # Benchmark Reproducibility Generated at: `2026-06-16` -Commit: `09518a4d8637c430311d50a4249913187dbc5057` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe` +Commit: `391821240ba14f088d0fbe12dd5f1fdca7917a39` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `24` - missing artifacts: `0` -- source contract sha256: `f48a71e36bdb` -- archive sha256: `00151114ce7f` +- source contract sha256: `9fde13846a41` +- archive sha256: `83ece575ef74` - output cases: `5` - disclosed failure cases: `3` - reproduction commands: `22` @@ -22,8 +22,8 @@ Evidence bundle SHA256: `9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8 - world-class ready: `false` - world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `36` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | @@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `24` / `24` -- sha256: `9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe` +- sha256: `5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8` ## Methodology Sections @@ -72,16 +73,16 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `30b43454ef2a` | +| output_execution | `reports/output_execution_runs.json` | present | `56430f01ba19` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `97f9ba949c23` | -| trust_report | `reports/security_trust_report.json` | present | `491354ae881f` | -| python_compatibility | `reports/python_compatibility.json` | present | `be0b8550a440` | -| registry_audit | `reports/registry_audit.json` | present | `870c455a5cd2` | -| package_verification | `reports/package_verification.json` | present | `c4acb824482d` | -| install_simulation | `reports/install_simulation.json` | present | `28ceb014e202` | +| trust_report | `reports/security_trust_report.json` | present | `a3a6d4e6809e` | +| python_compatibility | `reports/python_compatibility.json` | present | `e53158aa045e` | +| registry_audit | `reports/registry_audit.json` | present | `d6b523cc5cbb` | +| package_verification | `reports/package_verification.json` | present | `ae8ac3872be3` | +| install_simulation | `reports/install_simulation.json` | present | `3e27a6f7678e` | | skill_os2_audit | `reports/skill_os2_audit.json` | present | `57536bc67370` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `76a3f8e2b12b` | | world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `1cb74eaa978b` | diff --git a/reports/context_budget.json b/reports/context_budget.json index b4075fb8..469ba511 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1376201, + "other_text_tokens": 1377236, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1376968, - "deferred_resource_tokens": 436849, + "estimated_total_text_tokens": 1378003, + "deferred_resource_tokens": 436837, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110 + "estimated_tokens": 387626, + "file_count": 111 }, { "path": "references", @@ -31,8 +31,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110 + "estimated_tokens": 387626, + "file_count": 111 } ], "deferred_resource_governance": { @@ -54,14 +54,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110, + "estimated_tokens": 387626, + "file_count": 111, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 563, + "relevant_file_count": 565, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 135.4 diff --git a/reports/context_budget.md b/reports/context_budget.md index cff507d4..ee8b248b 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 960 | 767 | 436849 | `governed` | scripts:387638 | 135.4 | - | ok | +| root | `.` | `production` | 1000 | 960 | 767 | 436837 | `governed` | scripts:387626 | 135.4 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 3f45d524..88e78c34 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 960, "skill_body_tokens": 767, - "deferred_resource_tokens": 436849, + "deferred_resource_tokens": 436837, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110 + "estimated_tokens": 387626, + "file_count": 111 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110, + "estimated_tokens": 387626, + "file_count": 111, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 548c9a3f..59a2fd12 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -47,8 +47,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": true, - "actual": true, + "expected": false, + "actual": false, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -58,8 +58,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "09518a4d8637c430311d50a4249913187dbc5057", - "actual": "09518a4d8637c430311d50a4249913187dbc5057", + "expected": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "actual": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -71,30 +71,30 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -204,8 +204,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "09518a4d8637c430311d50a4249913187dbc5057", - "actual": "09518a4d8637c430311d50a4249913187dbc5057", + "expected": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "actual": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -217,30 +217,30 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -1322,7 +1322,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -1337,7 +1337,7 @@ "file_count": 222 } ], - "file_count": 399, + "file_count": 400, "folder_count": 4, "distribution": [ { @@ -1362,7 +1362,7 @@ }, { "label": "scripts", - "value": 110 + "value": 111 }, { "label": "evals", @@ -1410,7 +1410,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -1425,7 +1425,7 @@ "file_count": 222 } ], - "file_count": 399, + "file_count": 400, "folder_count": 4, "distribution": [ { @@ -1450,7 +1450,7 @@ }, { "label": "scripts", - "value": 110 + "value": 111 }, { "label": "evals", @@ -2025,13 +2025,13 @@ "label": "Skill OS 2.0 review summary mirrors current evidence", "status": "pass", "expected": [ - "score `91`", + "score `89`", "`16` gates", - "`3` warnings", - "`28` declared internal modules", - "`82 / 82` CLI help smoke checks passing across `110` scripts", - "`617` zip entries", - "archive with `617` entries", + "`4` warnings", + "`29` declared internal modules", + "`82 / 82` CLI help smoke checks passing across `111` scripts", + "`618` zip entries", + "archive with `618` entries", "`12` installer permission checks enforced", "`0` permission failures", "`24` required artifacts", diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 24528677..ac298757 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 617, + "archive_entry_count": 618, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 5210d93d..2f5bbfa4 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.28, + "duration_ms": 26.79, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.19, + "duration_ms": 27.31, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.08, + "duration_ms": 26.66, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.33, + "duration_ms": 26.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.89, + "duration_ms": 26.62, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.56, + "duration_ms": 26.34, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.12, + "duration_ms": 26.51, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.4, + "duration_ms": 26.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.2, + "duration_ms": 26.44, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.1, + "duration_ms": 26.21, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index ea989863..0b1f8aa1 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 26.28 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 26.19 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 26.08 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.33 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 25.89 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.56 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.12 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.4 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.2 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.1 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.79 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 27.31 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 26.66 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.21 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 26.62 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.34 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.51 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.37 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.44 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.21 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index b600a680..796fc70f 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", - "archive_entry_count": 617, + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", + "archive_entry_count": 618, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index df06b5e6..088c4db6 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137` +- Archive SHA256: `83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 0eeeda29..f731c0b8 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 176, + "file_count": 177, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -118,6 +118,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/evidence_consistency_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/export_skill_ir.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 7a40edff..8cb1d252 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `176` +- files scanned: `177` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index b0faf6c9..4007ecc1 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index e49d7504..28f79aa9 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086` -- Archive SHA256: `00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137` +- Package SHA256: `9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea` +- Archive SHA256: `83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 3b3c6448..698405fa 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -670,28 +670,28 @@
审查结论 review - Score 91/100 + Score 89/100

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

110 scripts scanned; secrets found

Py Compat0

176 files scanned for Python 3.11

Arch Debt0

899 largest lines; 9 watchlist; 64 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

173 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

617 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

111 scripts scanned; secrets found

Py Compat0

177 files scanned for Python 3.11

Arch Debt0

899 largest lines; 8 watchlist; 64 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

173 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

618 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 960/1000; deferred 436849/120000; top deferred scripts 387638; resource governance governed; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 110 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 176 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

173 Python files; 0 hotspots; 9 watchlist files; 0 blockers; largest 899 lines; 64 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
关注

上下文

initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 111 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 177 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

174 Python files; 0 hotspots; 8 watchlist files; 0 blockers; largest 899 lines; 64 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 2 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

阻断事项

无。

-

关注事项

+

关注事项

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据;提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  2. python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  3. Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

上下文

压缩或拆分高成本 deferred resources,保留最小可路由上下文。

初始加载可以安全,但后续 references、scripts、evals 体量过大时,reviewer 仍需要看到维护和读取成本。
修复位置
SKILL.md + references/ + scripts/ + evals/
验证命令
python3 scripts/render_context_reports.py
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据;提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  2. python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  3. Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
@@ -743,17 +743,17 @@
-

上下文

initial load 960/1000; deferred 436849/120000; top deferred scripts 387638; resource governance governed; quality density 135.4

+

上下文

initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
110
网络脚本
3
Help 失败
0
包体哈希
f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086
+

信任报告

Secret
0
脚本数
111
网络脚本
3
Help 失败
0
包体哈希
9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

-

Python 兼容

目标 Python
3.11
文件数
176
问题数
0
语法错误
0
F-string 3.11
0
+

Python 兼容

目标 Python
3.11
文件数
177
问题数
0
语法错误
0
F-string 3.11
0

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线;本地更高版本允许的新语法不能绕过兼容门禁。

@@ -781,7 +781,7 @@

人工批准

warning 可以被有边界地接受,但必须写入 reviewer、理由、范围和到期时间;blocker 与 world-class 完成证据不能通过 waiver 变成通过。

-

批准概况

0 active waivers; 1 warning gates still need reviewer decision

+

批准概况

0 active waivers; 2 warning gates still need reviewer decision

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0

批准候选

@@ -806,8 +806,8 @@
-

公开声明

本地复现
发布锁
可公开声明
声明阻断
4
Provider 证据
人审完成
世界级就绪
-

声明阻断

+

公开声明

本地复现
发布锁
可公开声明
声明阻断
5
Provider 证据
人审完成
世界级就绪
+

声明阻断

@@ -827,12 +827,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
617
失败数
0
警告数
0
归档哈希
00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137
+

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
618
失败数
0
警告数
0
归档哈希
83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5
diff --git a/reports/review-studio.json b/reports/review-studio.json index 5a20c2f6..931d7eb2 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -4,11 +4,11 @@ "skill_dir": ".", "summary": { "decision": "review", - "world_class_score": 91, + "world_class_score": 89, "gate_count": 16, "blocker_count": 0, - "warning_count": 3, - "action_count": 3, + "warning_count": 4, + "action_count": 4, "annotation_count": 0, "open_annotation_count": 0, "open_annotation_blocker_count": 0, @@ -42,8 +42,8 @@ { "key": "context-budget", "label": "上下文", - "status": "pass", - "detail": "initial load 960/1000; deferred 436849/120000; top deferred scripts 387638; resource governance governed; quality density 135.4", + "status": "warn", + "detail": "initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 110 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 111 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -67,7 +67,7 @@ "key": "python-compat", "label": "Python 兼容", "status": "pass", - "detail": "Python 3.11; 176 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "detail": "Python 3.11; 177 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", "evidence": "reports/python_compatibility.json", "link": "python_compatibility.md" }, @@ -75,7 +75,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "173 Python files; 0 hotspots; 9 watchlist files; 0 blockers; largest 899 lines; 64 CLI handlers; 18 in entrypoint", + "detail": "174 Python files; 0 hotspots; 8 watchlist files; 0 blockers; largest 899 lines; 64 CLI handlers; 18 in entrypoint", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md" }, @@ -115,7 +115,7 @@ "key": "review-waivers", "label": "人工批准", "status": "warn", - "detail": "0 active waivers; 1 warning gates still need reviewer decision", + "detail": "0 active waivers; 2 warning gates still need reviewer decision", "evidence": "reports/review_waivers.json", "link": "review_waivers.md" }, @@ -154,11 +154,19 @@ "evidence": "reports/output_quality_scorecard.json", "link": "output_quality_scorecard.md" }, + { + "key": "context-budget", + "label": "上下文", + "status": "warn", + "detail": "initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4", + "evidence": "reports/context_budget.json", + "link": "context_budget.md" + }, { "key": "review-waivers", "label": "人工批准", "status": "warn", - "detail": "0 active waivers; 1 warning gates still need reviewer decision", + "detail": "0 active waivers; 2 warning gates still need reviewer decision", "evidence": "reports/review_waivers.json", "link": "review_waivers.md" }, @@ -235,6 +243,53 @@ "verification_command": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review", "evidence_steps": [] }, + { + "gate_key": "context-budget", + "label": "上下文", + "status": "warn", + "priority": "warning", + "summary": "压缩或拆分高成本 deferred resources,保留最小可路由上下文。", + "why": "初始加载可以安全,但后续 references、scripts、evals 体量过大时,reviewer 仍需要看到维护和读取成本。", + "source_fix": "SKILL.md + references/ + scripts/ + evals/", + "source_refs": [ + { + "path": "SKILL.md", + "label": "entrypoint", + "kind": "source", + "line": 9, + "exists": true, + "link": "../SKILL.md" + }, + { + "path": "reports/context_budget.md", + "label": "context budget", + "kind": "report", + "line": 1, + "exists": true, + "link": "context_budget.md" + }, + { + "path": "scripts/resource_boundary_check.py", + "label": "resource boundary checker", + "kind": "source", + "line": 44, + "exists": true, + "link": "../scripts/resource_boundary_check.py" + }, + { + "path": "references/skill-engineering-method.md", + "label": "skill engineering method", + "kind": "method", + "line": 208, + "exists": true, + "link": "../references/skill-engineering-method.md" + } + ], + "evidence": "reports/context_budget.json", + "evidence_link": "context_budget.md", + "verification_command": "python3 scripts/render_context_reports.py", + "evidence_steps": [] + }, { "gate_key": "review-waivers", "label": "人工批准", @@ -1193,7 +1248,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -1208,7 +1263,7 @@ "file_count": 222 } ], - "file_count": 399, + "file_count": 400, "folder_count": 4, "distribution": [ { @@ -1233,7 +1288,7 @@ }, { "label": "scripts", - "value": 110 + "value": 111 }, { "label": "evals", @@ -1363,7 +1418,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -1676,11 +1731,11 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", + "evidence_bundle_sha256": "d4d58b9557d03656a800aad0acee41d5434db006ebbbff2645fbbab80962cc27", "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "output_case_count": 5, @@ -1700,11 +1755,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 20 }, - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1788,9 +1843,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 197, - "script_count": 110, - "internal_module_count": 28, + "scanned_files": 198, + "script_count": 111, + "internal_module_count": 29, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1808,8 +1863,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 197, - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_hash_file_count": 198, + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" }, "skill_atlas": { "skill_count": 12, @@ -1847,8 +1902,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, "compatibility": { "openai": "pass", @@ -1879,7 +1934,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1895,8 +1950,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", - "archive_entry_count": 617, + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", + "archive_entry_count": 618, "failure_count": 0, "warning_count": 0 }, @@ -1907,7 +1962,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 617, + "archive_entry_count": 618, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1974,12 +2029,12 @@ { "field": "archive_sha256", "from": "", - "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "to": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "to": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, @@ -4346,7 +4401,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.28, + "duration_ms": 26.59, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4374,7 +4429,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.19, + "duration_ms": 26.1, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4397,7 +4452,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.08, + "duration_ms": 26.12, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4425,7 +4480,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.33, + "duration_ms": 28.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4448,7 +4503,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.89, + "duration_ms": 29.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4476,7 +4531,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.56, + "duration_ms": 28.48, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4499,7 +4554,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.12, + "duration_ms": 26.51, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4526,7 +4581,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.4, + "duration_ms": 25.88, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4549,7 +4604,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.2, + "duration_ms": 25.99, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4578,7 +4633,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.1, + "duration_ms": 26.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5299,21 +5354,34 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 20, + "sample": [ + " M reports/adoption_drift_report.json", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md", + " M reports/python_compatibility.json", + " M reports/python_compatibility.md" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", + "evidence_bundle_sha256": "d4d58b9557d03656a800aad0acee41d5434db006ebbbff2645fbbab80962cc27", "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", "output_case_count": 5, @@ -5333,14 +5401,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 20 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -5349,10 +5418,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "ready": false, + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -5360,7 +5429,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe" + "sha256": "d4d58b9557d03656a800aad0acee41d5434db006ebbbff2645fbbab80962cc27" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5433,8 +5502,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7964, - "sha256": "30b43454ef2a4802b13d554100789b1c9c7a1baec9bebcd6fc8cf0ea9830ba41" + "bytes": 7967, + "sha256": "2ee3f1028f48673cddeb6242eb30aee3db1eccf26c44499744f3031d6d3520d7" }, { "label": "blind_review", @@ -5475,8 +5544,8 @@ "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 22887, - "sha256": "be0b8550a4404d85a044b3ae612b39d94e8b493723e7aaf02386cace291b8262" + "bytes": 23022, + "sha256": "e53158aa045e19f51cbf61f4cb8b299c9ac2babeaacd5441ecb911eb99d8089d" }, { "label": "registry_audit", @@ -5824,7 +5893,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "110 scripts; secrets 0; help failures 0", + "current": "111 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -5898,7 +5967,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 617; install failures 0", + "current": "archive entries 618; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -11391,9 +11460,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 197, - "script_count": 110, - "internal_module_count": 28, + "scanned_files": 198, + "script_count": 111, + "internal_module_count": 29, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -11411,8 +11480,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 197, - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_hash_file_count": 198, + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" }, "failures": [], "warnings": [], @@ -11605,6 +11674,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/evidence_consistency_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/export_skill_ir.py", "interface": "cli", @@ -13010,7 +13093,7 @@ "checked_count": 82, "passed_count": 82, "failed_count": 0, - "skipped_count": 28, + "skipped_count": 29, "failed_scripts": [], "results": [ { @@ -13835,6 +13918,10 @@ } ], "skipped": [ + { + "path": "scripts/evidence_consistency_world_class.py", + "reason": "internal module" + }, { "path": "scripts/html_rendering.py", "reason": "internal module" @@ -14138,7 +14225,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 176, + "file_count": 177, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -14251,6 +14338,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/evidence_consistency_world_class.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/export_skill_ir.py", "ok": true, @@ -15224,10 +15317,10 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 173, - "script_file_count": 110, + "python_file_count": 174, + "script_file_count": 111, "test_file_count": 63, - "internal_module_count": 31, + "internal_module_count": 32, "cli_script_count": 81, "command_handler_count": 64, "entrypoint_command_handler_count": 18, @@ -15236,7 +15329,7 @@ "watch_line_threshold": 720, "block_line_threshold": 1500, "largest_file_lines": 899, - "watchlist_count": 9, + "watchlist_count": 8, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" @@ -15249,13 +15342,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 859, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 810, @@ -15325,6 +15411,13 @@ "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/render_review_viewer.py", + "lines": 685, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Split viewer data assembly from HTML section rendering." } ], "watchlist": [ @@ -15335,13 +15428,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 859, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/yao_cli_parser.py", "lines": 810, @@ -15402,21 +15488,23 @@ "context_budget": { "ok": true, "failures": [], - "warnings": [], + "warnings": [ + "Deferred resource footprint is high: 436837 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified." + ], "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1376201, + "other_text_tokens": 1376303, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1376968, - "deferred_resource_tokens": 436849, + "estimated_total_text_tokens": 1377070, + "deferred_resource_tokens": 436837, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110 + "estimated_tokens": 387626, + "file_count": 111 }, { "path": "references", @@ -15432,37 +15520,38 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110 + "estimated_tokens": 387626, + "file_count": 111 } ], "deferred_resource_governance": { - "status": "governed", + "status": "needs-review", "large_dir_count": 1, - "governed_large_dir_count": 1, + "governed_large_dir_count": 0, "directories": [ { - "status": "governed", + "status": "needs-review", "evidence": [ "reports/security_trust_report.json", "reports/architecture_maintainability.json", "reports/python_compatibility.json" ], "reasons": [ - "trust report covers scripts", "architecture report has no script hotspots or blockers", "Python compatibility report has no issues" ], - "missing": [], + "missing": [ + "trust report covers scripts" + ], "path": "scripts", - "estimated_tokens": 387638, - "file_count": 110, + "estimated_tokens": 387626, + "file_count": 111, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], - "summary": "Large deferred resources are indexed and backed by evidence." + "summary": "One or more large deferred resource directories still need explicit governance evidence." }, - "relevant_file_count": 563, + "relevant_file_count": 565, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 135.4 @@ -17330,7 +17419,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-15T16:57:40Z", + "generated_at": "2026-06-15T17:03:40Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -20445,8 +20534,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, "compatibility": { "openai": "pass", @@ -20477,7 +20566,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -20502,7 +20591,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, @@ -20525,8 +20614,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", - "archive_entry_count": 617, + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", + "archive_entry_count": 618, "failure_count": 0, "warning_count": 0 }, @@ -21191,7 +21280,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 617, + "archive_entry_count": 618, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -21533,12 +21622,12 @@ { "field": "archive_sha256", "from": "", - "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "to": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "to": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 44cc0574..c48a2612 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -513,7 +513,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -528,7 +528,7 @@ "file_count": 222 } ], - "file_count": 399, + "file_count": 400, "folder_count": 4, "distribution": [ { @@ -553,7 +553,7 @@ }, { "label": "scripts", - "value": 110 + "value": 111 }, { "label": "evals", @@ -683,7 +683,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -996,13 +996,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "evidence_bundle_sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -1020,11 +1020,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 36 }, - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1108,9 +1108,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 197, - "script_count": 110, - "internal_module_count": 28, + "scanned_files": 198, + "script_count": 111, + "internal_module_count": 29, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1128,8 +1128,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 197, - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_hash_file_count": 198, + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" }, "skill_atlas": { "skill_count": 12, @@ -1167,8 +1167,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, "compatibility": { "openai": "pass", @@ -1199,7 +1199,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1215,8 +1215,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", - "archive_entry_count": 617, + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", + "archive_entry_count": 618, "failure_count": 0, "warning_count": 0 }, @@ -1227,7 +1227,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 617, + "archive_entry_count": 618, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1294,12 +1294,12 @@ { "field": "archive_sha256", "from": "", - "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "to": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "to": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 95738cef..c03e8ad3 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 197, - "script_count": 110, - "internal_module_count": 28, + "scanned_files": 198, + "script_count": 111, + "internal_module_count": 29, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 197, - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_hash_file_count": 198, + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" }, "failures": [], "warnings": [], @@ -216,6 +216,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/evidence_consistency_world_class.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/export_skill_ir.py", "interface": "cli", @@ -1621,7 +1635,7 @@ "checked_count": 82, "passed_count": 82, "failed_count": 0, - "skipped_count": 28, + "skipped_count": 29, "failed_scripts": [], "results": [ { @@ -2446,6 +2460,10 @@ } ], "skipped": [ + { + "path": "scripts/evidence_consistency_world_class.py", + "reason": "internal module" + }, { "path": "scripts/html_rendering.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 91164b78..1b09bdb8 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `197` -- Scripts: `110` -- Internal script modules: `28` +- Scanned files: `198` +- Scripts: `111` +- Internal script modules: `29` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `197` -- Package SHA256: `f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086` +- Package hash files: `198` +- Package SHA256: `9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea` ## Failures @@ -75,6 +75,7 @@ | scripts/cross_packager.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/diff_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/emit_telemetry_event.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/evidence_consistency_world_class.py | internal-module | True | False | False | False | False | False | False | Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks. | | scripts/export_skill_ir.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/github_benchmark_scan.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/governance_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-interpretation.html b/reports/skill-interpretation.html index 3ab299ee..669c5ace 100644 --- a/reports/skill-interpretation.html +++ b/reports/skill-interpretation.html @@ -930,7 +930,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布Asset Distribution399项399 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+
资产分布Asset Distribution400项400 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 4bf04659..a3a7a072 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -513,7 +513,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -528,7 +528,7 @@ "file_count": 222 } ], - "file_count": 399, + "file_count": 400, "folder_count": 4, "distribution": [ { @@ -553,7 +553,7 @@ }, { "label": "scripts", - "value": 110 + "value": 111 }, { "label": "evals", @@ -687,7 +687,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -1000,13 +1000,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "evidence_bundle_sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -1024,11 +1024,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 36 }, - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1112,9 +1112,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 197, - "script_count": 110, - "internal_module_count": 28, + "scanned_files": 198, + "script_count": 111, + "internal_module_count": 29, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1132,8 +1132,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 197, - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_hash_file_count": 198, + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" }, "skill_atlas": { "skill_count": 12, @@ -1171,8 +1171,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, "compatibility": { "openai": "pass", @@ -1203,7 +1203,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1219,8 +1219,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", - "archive_entry_count": 617, + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", + "archive_entry_count": 618, "failure_count": 0, "warning_count": 0 }, @@ -1231,7 +1231,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 617, + "archive_entry_count": 618, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1298,12 +1298,12 @@ { "field": "archive_sha256", "from": "", - "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "to": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "to": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 2de15f06..71d9e581 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -37,7 +37,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Local Install Sync Preflight v0 so source-to-local and source-to-active install syncs rebuild the package first, run install simulation, enforce installer permission coverage, and refuse to copy files before any destructive sync when the package is not install-ready. - Atlas Scope Policy v0 so examples, evolution snapshots, embedded generated skills, and validator fixtures remain visible in the full portfolio report without polluting release-actionable gates. - Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision. -- Review Studio now avoids over-claiming release readiness when blind A/B adjudication and world-class evidence are still pending: the root Meta Skill is in `review` with score `91`, no blockers, three warnings, and explicit actions for Output Lab reviewer adjudication, waiver handling, and world-class evidence completion. +- Review Studio now avoids over-claiming release readiness when blind A/B adjudication, context-budget pressure, waiver handling, and world-class evidence are still pending: the root Meta Skill is in `review` with score `89`, no blockers, four warnings, and explicit actions for Output Lab reviewer adjudication, context-budget follow-up, waiver handling, and world-class evidence completion. - Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning. - Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case. - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. @@ -91,7 +91,7 @@ Next move: run provider-backed holdout cases with real credentials and one real ### 3. Review Studio is unified and now has reviewer actions plus annotations -The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, world-class evidence, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. The current root report is intentionally not a clean pass: decision `review`, score `91`, `16` gates, `0` blockers, `3` warnings, `3` actions, and `0` open annotation blockers. The warnings are useful evidence: the automated Output Eval Lab is strong, but the blind A/B adjudication still has `5` pending reviewer decisions and the world-class ledger still has `4` pending external or human evidence entries, so the package should not claim fully reviewed or world-class status. +The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, world-class evidence, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. The current root report is intentionally not a clean pass: decision `review`, score `89`, `16` gates, `0` blockers, `4` warnings, `4` actions, and `0` open annotation blockers. The warnings are useful evidence: the automated Output Eval Lab is strong, but the blind A/B adjudication still has `5` pending reviewer decisions, the context budget is close to its limit, and the world-class ledger still has `4` pending external or human evidence entries, so the package should not claim fully reviewed or world-class status. Next move: add richer source-line anchors inside generated reports, record real reviewer annotations during the next human review pass, and close the provider, native-permission, real-client telemetry, and human review evidence entries in the world-class ledger. @@ -114,19 +114,19 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `28` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `82 / 82` CLI help smoke checks passing across `110` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `29` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `82 / 82` CLI help smoke checks passing across `111` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `617` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `617` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `618` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `618` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | | Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0, and world-class evidence gaps must be closed with accepted evidence rather than waived as completed | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | -| Review Studio | decision `review`, world-class score `91`, `16` gates, `0` blockers, `3` warnings, `3` review actions, `0` open annotation blockers | +| Review Studio | decision `review`, world-class score `89`, `16` gates, `0` blockers, `4` warnings, `4` review actions, `0` open annotation blockers | | Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps | | World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts | | World-Class Evidence Ledger | `4` evidence entries remain pending: `1` human-required and `3` external-required; anti-overclaim guards block planned work, metadata fallbacks, pending review, and local command runners from counting as completion | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 4545e451..5786de6f 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -930,7 +930,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布Asset Distribution399项399 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
+
资产分布Asset Distribution400项400 itemsSKILL.mdSKILL.mdREADME.mdREADME.mdagents/interface.yamlagents/interface.yamlmanifest.jsonmanifest.jsonreferencesreferencesscriptsscripts
资产分布图展示当前包体的文件和目录重心。The asset distribution chart shows where files and directories are concentrated.
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index dc567230..d66178e6 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -512,7 +512,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -527,7 +527,7 @@ "file_count": 222 } ], - "file_count": 399, + "file_count": 400, "folder_count": 4, "distribution": [ { @@ -552,7 +552,7 @@ }, { "label": "scripts", - "value": 110 + "value": 111 }, { "label": "evals", @@ -682,7 +682,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 110 + "file_count": 111 }, { "path": "evals", @@ -995,13 +995,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "9bc8af4e6aa04e1f61bc42b432335e0f3500515d3e3d59a6c430ce8c1177fbbe", - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "evidence_bundle_sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -1019,11 +1019,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 36 }, - "commit": "09518a4d8637c430311d50a4249913187dbc5057", + "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -1107,9 +1107,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 197, - "script_count": 110, - "internal_module_count": 28, + "scanned_files": 198, + "script_count": 111, + "internal_module_count": 29, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1127,8 +1127,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 197, - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "package_hash_file_count": 198, + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" }, "skill_atlas": { "skill_count": 12, @@ -1166,8 +1166,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "package_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, "compatibility": { "openai": "pass", @@ -1198,7 +1198,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1214,8 +1214,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", - "archive_entry_count": 617, + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", + "archive_entry_count": 618, "failure_count": 0, "warning_count": 0 }, @@ -1226,7 +1226,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 617, + "archive_entry_count": 618, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1293,12 +1293,12 @@ { "field": "archive_sha256", "from": "", - "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "to": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "to": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 80ab3a7c..452e124e 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "110 scripts; secrets 0; help failures 0", + "current": "111 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -204,7 +204,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 617; install failures 0", + "current": "archive entries 618; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -245,7 +245,7 @@ "label": "Review Studio", "status": "pass", "objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.", - "current": "16 gates; decision review; warnings 3", + "current": "16 gates; decision review; warnings 4", "command": "python3 scripts/yao.py review-studio .", "test": "python3 tests/verify_review_studio.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index d43928ef..2347d6ba 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,10 +26,10 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 110 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 111 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 617; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | -| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | +| Registry Distribution | `pass` | archive entries 618; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Review Studio | `pass` | 16 gates; decision review; warnings 4 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | ## Recommended PR Coverage diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index ea4a7ffe..76f81846 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137" + "to": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086" + "to": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea" } ] }, diff --git a/scripts/evidence_consistency_world_class.py b/scripts/evidence_consistency_world_class.py new file mode 100644 index 00000000..8f629d1b --- /dev/null +++ b/scripts/evidence_consistency_world_class.py @@ -0,0 +1,197 @@ +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to isolate world-class evidence workflow consistency checks." + + +def keyed_items(payload: dict[str, Any], collection_key: str) -> dict[str, dict[str, Any]]: + collection = payload.get(collection_key) + if not isinstance(collection, list): + return {} + keyed: dict[str, dict[str, Any]] = {} + for item in collection: + if not isinstance(item, dict): + continue + key = item.get("key") or item.get("evidence_key") + if isinstance(key, str): + keyed[key] = item + return keyed + + +def command_key_set(item: dict[str, Any]) -> set[str]: + commands = item.get("commands") + if isinstance(commands, dict): + return {key for key, value in commands.items() if isinstance(key, str) and value} + if isinstance(commands, list): + keys: set[str] = set() + for command in commands: + if isinstance(command, dict) and isinstance(command.get("key"), str) and command.get("command"): + keys.add(command["key"]) + return keys + return set() + + +def world_class_review_action_steps(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]: + actions = review_studio.get("review_actions") + if not isinstance(actions, list): + return {} + for action in actions: + if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence": + continue + steps = action.get("evidence_steps") + if not isinstance(steps, list): + return {} + keyed: dict[str, dict[str, Any]] = {} + for step in steps: + if isinstance(step, dict) and isinstance(step.get("key"), str): + keyed[step["key"]] = step + return keyed + return {} + + +def command_groups_present(command_keys: set[str]) -> dict[str, bool]: + return { + "prepare_submission": "prepare_submission" in command_keys, + "validate_intake": "validate_intake" in command_keys, + "submission_review": bool({"submission_review", "review_queue"} & command_keys), + "refresh_ledger": "refresh_ledger" in command_keys, + "guard_claim": "guard_claim" in command_keys, + } + + +def has_next_action(item: dict[str, Any]) -> bool: + for key in ["next_action", "audit_next_action"]: + value = item.get(key) + if isinstance(value, str) and value.strip(): + return True + return False + + +def command_expectations(keys: list[str]) -> dict[str, dict[str, dict[str, bool]]]: + expected_group = { + "prepare_submission": True, + "validate_intake": True, + "submission_review": True, + "refresh_ledger": True, + "guard_claim": True, + } + return { + key: { + "intake": dict(expected_group), + "operator_runbook": dict(expected_group), + "review_studio": dict(expected_group), + } + for key in keys + } + + +def build_world_class_workflow_check( + *, + ledger: dict[str, Any], + world_class_plan: dict[str, Any], + world_class_intake: dict[str, Any], + world_class_submission_review: dict[str, Any], + world_class_operator_runbook: dict[str, Any], + review_studio: dict[str, Any], + report_paths: dict[str, str], +) -> dict[str, Any]: + ledger_summary = ledger.get("summary") if isinstance(ledger.get("summary"), dict) else {} + plan_summary = world_class_plan.get("summary") if isinstance(world_class_plan.get("summary"), dict) else {} + intake_summary = world_class_intake.get("summary") if isinstance(world_class_intake.get("summary"), dict) else {} + submission_review_summary = ( + world_class_submission_review.get("summary") + if isinstance(world_class_submission_review.get("summary"), dict) + else {} + ) + operator_runbook_summary = ( + world_class_operator_runbook.get("summary") + if isinstance(world_class_operator_runbook.get("summary"), dict) + else {} + ) + + ledger_items = keyed_items(ledger, "entries") + plan_tasks = keyed_items(world_class_plan, "tasks") + intake_checklist = keyed_items(world_class_intake, "operator_checklist") + submission_review_items = keyed_items(world_class_submission_review, "items") + operator_runbook_items = keyed_items(world_class_operator_runbook, "items") + review_action_steps = world_class_review_action_steps(review_studio) + pending_keys = sorted(key for key, item in ledger_items.items() if item.get("status") != "accepted") + + actual_command_groups = { + key: { + "intake": command_groups_present(command_key_set(intake_checklist.get(key, {}))), + "operator_runbook": command_groups_present(command_key_set(operator_runbook_items.get(key, {}))), + "review_studio": command_groups_present(command_key_set(review_action_steps.get(key, {}))), + } + for key in pending_keys + } + expected = { + "keys": pending_keys, + "pending_count": ledger_summary.get("pending_count"), + "human_pending_count": ledger_summary.get("human_pending_count"), + "external_pending_count": ledger_summary.get("external_pending_count"), + "source_check_count": ledger_summary.get("source_check_count"), + "source_pass_count": ledger_summary.get("source_pass_count"), + "source_blocked_count": ledger_summary.get("source_blocked_count"), + "plan_keys": pending_keys, + "intake_keys": pending_keys, + "submission_review_keys": pending_keys, + "operator_runbook_keys": pending_keys, + "review_studio_keys": pending_keys, + "intake_ready_to_claim_world_class": False, + "submission_review_ready_to_claim_world_class": False, + "submission_review_counts_as_completion": False, + "operator_runbook_ready_to_claim_world_class": False, + "operator_runbook_counts_as_completion": False, + "next_actions_present": {key: True for key in pending_keys}, + "commands": command_expectations(pending_keys), + } + actual = { + "keys": pending_keys, + "pending_count": plan_summary.get("task_count"), + "human_pending_count": plan_summary.get("human_task_count"), + "external_pending_count": plan_summary.get("external_task_count"), + "source_check_count": submission_review_summary.get("source_check_count"), + "source_pass_count": submission_review_summary.get("source_pass_count"), + "source_blocked_count": submission_review_summary.get("source_blocked_count"), + "plan_keys": sorted(plan_tasks), + "intake_keys": sorted(intake_checklist), + "submission_review_keys": sorted(submission_review_items), + "operator_runbook_keys": sorted(operator_runbook_items), + "review_studio_keys": sorted(review_action_steps), + "intake_ready_to_claim_world_class": intake_summary.get("ready_to_claim_world_class"), + "submission_review_ready_to_claim_world_class": submission_review_summary.get("ready_to_claim_world_class"), + "submission_review_counts_as_completion": submission_review_summary.get( + "review_counts_submission_as_completion" + ), + "operator_runbook_ready_to_claim_world_class": operator_runbook_summary.get("ready_to_claim_world_class"), + "operator_runbook_counts_as_completion": operator_runbook_summary.get("runbook_counts_as_completion"), + "next_actions_present": { + key: all( + has_next_action(collection.get(key, {})) + for collection in [plan_tasks, intake_checklist, submission_review_items, review_action_steps] + ) + for key in pending_keys + }, + "commands": actual_command_groups, + } + return { + "key": "world-class-evidence-workflow-coverage", + "label": "World-class evidence workflows cover every pending ledger entry", + "status": "pass" if expected == actual else "fail", + "expected": expected, + "actual": actual, + "paths": [ + report_paths["world_class_ledger"], + report_paths["world_class_plan"], + report_paths["world_class_intake"], + report_paths["world_class_submission_review"], + report_paths["world_class_operator_runbook"], + report_paths["review_studio"], + ], + "detail": ( + "Every pending world-class evidence key must have matching plan, intake, submission review, " + "operator runbook, and Review Studio actions without counting planned work as completion." + ), + } diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index 2f24a493..345d64d9 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -6,6 +6,8 @@ from datetime import date from pathlib import Path from typing import Any +from evidence_consistency_world_class import build_world_class_workflow_check + ROOT = Path(__file__).resolve().parent.parent SCRIPT_INTERFACE = "cli" @@ -149,69 +151,6 @@ def as_int(value: Any) -> int | None: return None -def keyed_items(payload: dict[str, Any], collection_key: str) -> dict[str, dict[str, Any]]: - collection = payload.get(collection_key) - if not isinstance(collection, list): - return {} - keyed: dict[str, dict[str, Any]] = {} - for item in collection: - if not isinstance(item, dict): - continue - key = item.get("key") or item.get("evidence_key") - if isinstance(key, str): - keyed[key] = item - return keyed - - -def command_key_set(item: dict[str, Any]) -> set[str]: - commands = item.get("commands") - if isinstance(commands, dict): - return {key for key, value in commands.items() if isinstance(key, str) and value} - if isinstance(commands, list): - keys: set[str] = set() - for command in commands: - if isinstance(command, dict) and isinstance(command.get("key"), str) and command.get("command"): - keys.add(command["key"]) - return keys - return set() - - -def world_class_review_action_steps(review_studio: dict[str, Any]) -> dict[str, dict[str, Any]]: - actions = review_studio.get("review_actions") - if not isinstance(actions, list): - return {} - for action in actions: - if not isinstance(action, dict) or action.get("gate_key") != "world-class-evidence": - continue - steps = action.get("evidence_steps") - if not isinstance(steps, list): - return {} - keyed: dict[str, dict[str, Any]] = {} - for step in steps: - if isinstance(step, dict) and isinstance(step.get("key"), str): - keyed[step["key"]] = step - return keyed - return {} - - -def command_groups_present(command_keys: set[str]) -> dict[str, bool]: - return { - "prepare_submission": "prepare_submission" in command_keys, - "validate_intake": "validate_intake" in command_keys, - "submission_review": bool({"submission_review", "review_queue"} & command_keys), - "refresh_ledger": "refresh_ledger" in command_keys, - "guard_claim": "guard_claim" in command_keys, - } - - -def has_next_action(item: dict[str, Any]) -> bool: - for key in ["next_action", "audit_next_action"]: - value = item.get(key) - if isinstance(value, str) and value.strip(): - return True - return False - - def add_check( checks: list[dict[str, Any]], *, @@ -332,10 +271,6 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: benchmark_summary = nested(benchmark, ["summary"], {}) adoption_summary = nested(adoption, ["summary"], {}) ledger_summary = nested(ledger, ["summary"], {}) - plan_summary = nested(world_class_plan, ["summary"], {}) - intake_summary = nested(world_class_intake, ["summary"], {}) - submission_review_summary = nested(world_class_submission_review, ["summary"], {}) - operator_runbook_summary = nested(world_class_operator_runbook, ["summary"], {}) coverage_summary = nested(coverage, ["summary"], {}) studio_summary = nested(review_studio, ["summary"], {}) package_summary = nested(package_verification, ["summary"], {}) @@ -585,130 +520,16 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: ], detail="The overclaim guard must scan package manifests, adapter metadata, security policy, and ledger surfaces before public readiness can be trusted.", ) - ledger_items = keyed_items(ledger, "entries") - plan_tasks = keyed_items(world_class_plan, "tasks") - intake_checklist = keyed_items(world_class_intake, "operator_checklist") - submission_review_items = keyed_items(world_class_submission_review, "items") - operator_runbook_items = keyed_items(world_class_operator_runbook, "items") - review_action_steps = world_class_review_action_steps(review_studio) - pending_world_class_keys = sorted( - key for key, item in ledger_items.items() if item.get("status") != "accepted" - ) - expected_command_groups = { - key: { - "intake": { - "prepare_submission": True, - "validate_intake": True, - "submission_review": True, - "refresh_ledger": True, - "guard_claim": True, - }, - "operator_runbook": { - "prepare_submission": True, - "validate_intake": True, - "submission_review": True, - "refresh_ledger": True, - "guard_claim": True, - }, - "review_studio": { - "prepare_submission": True, - "validate_intake": True, - "submission_review": True, - "refresh_ledger": True, - "guard_claim": True, - }, - } - for key in pending_world_class_keys - } - actual_command_groups = { - key: { - "intake": command_groups_present(command_key_set(intake_checklist.get(key, {}))), - "operator_runbook": command_groups_present(command_key_set(operator_runbook_items.get(key, {}))), - "review_studio": command_groups_present(command_key_set(review_action_steps.get(key, {}))), - } - for key in pending_world_class_keys - } - expected_world_class_workflow = { - "keys": pending_world_class_keys, - "pending_count": ledger_summary.get("pending_count") if isinstance(ledger_summary, dict) else None, - "human_pending_count": ledger_summary.get("human_pending_count") if isinstance(ledger_summary, dict) else None, - "external_pending_count": ledger_summary.get("external_pending_count") if isinstance(ledger_summary, dict) else None, - "source_check_count": ledger_summary.get("source_check_count") if isinstance(ledger_summary, dict) else None, - "source_pass_count": ledger_summary.get("source_pass_count") if isinstance(ledger_summary, dict) else None, - "source_blocked_count": ledger_summary.get("source_blocked_count") if isinstance(ledger_summary, dict) else None, - "plan_keys": pending_world_class_keys, - "intake_keys": pending_world_class_keys, - "submission_review_keys": pending_world_class_keys, - "operator_runbook_keys": pending_world_class_keys, - "review_studio_keys": pending_world_class_keys, - "intake_ready_to_claim_world_class": False, - "submission_review_ready_to_claim_world_class": False, - "submission_review_counts_as_completion": False, - "operator_runbook_ready_to_claim_world_class": False, - "operator_runbook_counts_as_completion": False, - "next_actions_present": {key: True for key in pending_world_class_keys}, - "commands": expected_command_groups, - } - actual_world_class_workflow = { - "keys": pending_world_class_keys, - "pending_count": plan_summary.get("task_count") if isinstance(plan_summary, dict) else None, - "human_pending_count": plan_summary.get("human_task_count") if isinstance(plan_summary, dict) else None, - "external_pending_count": plan_summary.get("external_task_count") if isinstance(plan_summary, dict) else None, - "source_check_count": submission_review_summary.get("source_check_count") - if isinstance(submission_review_summary, dict) - else None, - "source_pass_count": submission_review_summary.get("source_pass_count") - if isinstance(submission_review_summary, dict) - else None, - "source_blocked_count": submission_review_summary.get("source_blocked_count") - if isinstance(submission_review_summary, dict) - else None, - "plan_keys": sorted(plan_tasks), - "intake_keys": sorted(intake_checklist), - "submission_review_keys": sorted(submission_review_items), - "operator_runbook_keys": sorted(operator_runbook_items), - "review_studio_keys": sorted(review_action_steps), - "intake_ready_to_claim_world_class": intake_summary.get("ready_to_claim_world_class") - if isinstance(intake_summary, dict) - else None, - "submission_review_ready_to_claim_world_class": submission_review_summary.get("ready_to_claim_world_class") - if isinstance(submission_review_summary, dict) - else None, - "submission_review_counts_as_completion": submission_review_summary.get( - "review_counts_submission_as_completion" + checks.append( + build_world_class_workflow_check( + ledger=ledger, + world_class_plan=world_class_plan, + world_class_intake=world_class_intake, + world_class_submission_review=world_class_submission_review, + world_class_operator_runbook=world_class_operator_runbook, + review_studio=review_studio, + report_paths=REQUIRED_REPORTS, ) - if isinstance(submission_review_summary, dict) - else None, - "operator_runbook_ready_to_claim_world_class": operator_runbook_summary.get("ready_to_claim_world_class") - if isinstance(operator_runbook_summary, dict) - else None, - "operator_runbook_counts_as_completion": operator_runbook_summary.get("runbook_counts_as_completion") - if isinstance(operator_runbook_summary, dict) - else None, - "next_actions_present": { - key: all( - has_next_action(collection.get(key, {})) - for collection in [plan_tasks, intake_checklist, submission_review_items, review_action_steps] - ) - for key in pending_world_class_keys - }, - "commands": actual_command_groups, - } - compare_values( - checks, - key="world-class-evidence-workflow-coverage", - label="World-class evidence workflows cover every pending ledger entry", - expected=expected_world_class_workflow, - actual=actual_world_class_workflow, - paths=[ - REQUIRED_REPORTS["world_class_ledger"], - REQUIRED_REPORTS["world_class_plan"], - REQUIRED_REPORTS["world_class_intake"], - REQUIRED_REPORTS["world_class_submission_review"], - REQUIRED_REPORTS["world_class_operator_runbook"], - REQUIRED_REPORTS["review_studio"], - ], - detail="Every pending world-class evidence key must have matching plan, intake, submission review, operator runbook, and Review Studio actions without counting planned work as completion.", ) skill_os2_review = text_reports.get("skill_os2_review", "") ci_target_count = ci_default_target_count(skill_dir / "scripts" / "ci_test.py") diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py index 01e22d1d..6a4ec397 100644 --- a/tests/verify_trust_check.py +++ b/tests/verify_trust_check.py @@ -91,6 +91,7 @@ def main() -> None: "scripts/review_studio_formatting.py", "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", + "scripts/evidence_consistency_world_class.py", "scripts/skill_report_charts.py", "scripts/skill_report_i18n.py", "scripts/skill_report_layout.py", @@ -113,6 +114,7 @@ def main() -> None: assert "review_studio_formatting.py" not in warning_text, payload["warnings"] assert "review_studio_gates.py" not in warning_text, payload["warnings"] assert "review_studio_layout.py" not in warning_text, payload["warnings"] + assert "evidence_consistency_world_class.py" not in warning_text, payload["warnings"] assert "skill_report_charts.py" not in warning_text, payload["warnings"] assert "skill_report_i18n.py" not in warning_text, payload["warnings"] assert "skill_report_layout.py" not in warning_text, payload["warnings"]
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder