frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Govern context report freshness in release evidence

Browse Source
This commit is contained in:
yaojingang
2026-06-16 02:17:02 +08:00
parent ae9d65aeff
commit fc5f1dbca2
27 changed files with 206 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+2
View File
@@ -48,6 +48,7 @@ python3 scripts/render_world_class_submission_review.py . --generated-at "$GENER
python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT"
python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT"
python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT"
python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"
python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"
python3 scripts/render_skill_overview.py .
python3 scripts/render_skill_interpretation.py .
@@ -60,6 +61,7 @@ python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"
For final release evidence, commit source and generated package evidence first, then run the clean-lock reports from a clean worktree:
```bash
python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"
python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"
python3 scripts/render_skill_overview.py .
python3 scripts/render_skill_interpretation.py .
registry/index.json
+1 -1
View File
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
"package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c"
}
]
}
registry/packages/yao-meta-skill.json
+3 -3
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d",
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903"
"package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c",
"archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3"
},
"compatibility": {
"openai": "pass",
@@ -48,7 +48,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903",
"archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
reports/architecture_maintainability.json
+15 -8
View File
@@ -16,7 +16,7 @@
"watch_line_threshold": 720,
"block_line_threshold": 1500,
"largest_file_lines": 899,
"watchlist_count": 8,
"watchlist_count": 9,
"hotspot_count": 0,
"blocker_count": 0,
"decision": "pass"
@@ -43,6 +43,13 @@
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/render_evidence_consistency.py",
"lines": 804,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/skill_report_model.py",
"lines": 801,
@@ -85,13 +92,6 @@
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/render_evidence_consistency.py",
"lines": 707,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/apply_adaptation.py",
"lines": 706,
@@ -129,6 +129,13 @@
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/render_evidence_consistency.py",
"lines": 804,
"kind": "cli-script",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/skill_report_model.py",
"lines": 801,
reports/architecture_maintainability.md
+3 -2
View File
@@ -15,7 +15,7 @@ Generated at: `2026-06-16`
- command modules: `6`
- largest file lines: `899`
- watch threshold lines: `720`
- watchlist: `8`
- watchlist: `9`
- hotspots: `0`
- blockers: `0`
@@ -32,6 +32,7 @@ No file-size hotspots found.
| `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
| `scripts/yao_cli_parser.py` | `821` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
| `scripts/skill_report_layout.py` | `808` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
| `scripts/render_evidence_consistency.py` | `804` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
| `scripts/skill_report_model.py` | `801` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
| `tests/verify_review_studio.py` | `763` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
| `scripts/compile_skill.py` | `734` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
@@ -45,13 +46,13 @@ No file-size hotspots found.
| `tests/verify_yao_cli.py` | `899` | `test` | `pass` |
| `scripts/yao_cli_parser.py` | `821` | `internal-module` | `pass` |
| `scripts/skill_report_layout.py` | `808` | `internal-module` | `pass` |
| `scripts/render_evidence_consistency.py` | `804` | `cli-script` | `pass` |
| `scripts/skill_report_model.py` | `801` | `internal-module` | `pass` |
| `tests/verify_review_studio.py` | `763` | `test` | `pass` |
| `scripts/compile_skill.py` | `734` | `cli-script` | `pass` |
| `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` |
| `scripts/optimize_description.py` | `723` | `cli-script` | `pass` |
| `scripts/trust_check.py` | `714` | `internal-module` | `pass` |
| `scripts/render_evidence_consistency.py` | `707` | `cli-script` | `pass` |
| `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` |
| `scripts/review_studio_layout.py` | `694` | `internal-module` | `pass` |
reports/context_budget.json
+13 -16
View File
@@ -1,22 +1,20 @@
{
"ok": true,
"failures": [],
"warnings": [
"Deferred resource footprint is high: 442670 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified."
],
"warnings": [],
"stats": {
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 767,
"other_text_tokens": 1395152,
"other_text_tokens": 1404003,
"estimated_initial_load_tokens": 960,
"estimated_total_text_tokens": 1395919,
"deferred_resource_tokens": 442670,
"estimated_total_text_tokens": 1404770,
"deferred_resource_tokens": 444818,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 393459,
"estimated_tokens": 395607,
"file_count": 113
},
{
@@ -33,36 +31,35 @@
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 393459,
"estimated_tokens": 395607,
"file_count": 113
}
],
"deferred_resource_governance": {
"status": "needs-review",
"status": "governed",
"large_dir_count": 1,
"governed_large_dir_count": 0,
"governed_large_dir_count": 1,
"directories": [
{
"status": "needs-review",
"status": "governed",
"evidence": [
"reports/security_trust_report.json",
"reports/architecture_maintainability.json",
"reports/python_compatibility.json"
],
"reasons": [
"trust report covers scripts",
"architecture report has no script hotspots or blockers",
"Python compatibility report has no issues"
],
"missing": [
"trust report covers scripts"
],
"missing": [],
"path": "scripts",
"estimated_tokens": 393459,
"estimated_tokens": 395607,
"file_count": 113,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
"summary": "One or more large deferred resource directories still need explicit governance evidence."
"summary": "Large deferred resources are indexed and backed by evidence."
},
"relevant_file_count": 573,
"unused_resource_dirs": [],
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
| root | `.` | `production` | 1000 | 960 | 767 | 442670 | `needs-review` | scripts:393459 | 135.4 | - | ok |
| root | `.` | `production` | 1000 | 960 | 767 | 444818 | `governed` | scripts:395607 | 135.4 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
reports/context_budget_summary.json
+11 -14
View File
@@ -1,5 +1,5 @@
{
"generated_at": "2026-03-31",
"generated_at": "2026-06-16",
"targets": [
{
"label": "root",
@@ -8,47 +8,44 @@
"budget_limit": 1000,
"initial_tokens": 960,
"skill_body_tokens": 767,
"deferred_resource_tokens": 442670,
"deferred_resource_tokens": 444818,
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 393459,
"estimated_tokens": 395607,
"file_count": 113
}
],
"deferred_resource_governance": {
"status": "needs-review",
"status": "governed",
"large_dir_count": 1,
"governed_large_dir_count": 0,
"governed_large_dir_count": 1,
"directories": [
{
"status": "needs-review",
"status": "governed",
"evidence": [
"reports/security_trust_report.json",
"reports/architecture_maintainability.json",
"reports/python_compatibility.json"
],
"reasons": [
"trust report covers scripts",
"architecture report has no script hotspots or blockers",
"Python compatibility report has no issues"
],
"missing": [
"trust report covers scripts"
],
"missing": [],
"path": "scripts",
"estimated_tokens": 393459,
"estimated_tokens": 395607,
"file_count": 113,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
"summary": "One or more large deferred resource directories still need explicit governance evidence."
"summary": "Large deferred resources are indexed and backed by evidence."
},
"quality_density": 135.4,
"unused_resource_dirs": [],
"ok": true,
"warnings": [
"Deferred resource footprint is high: 442670 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified."
]
"warnings": []
},
{
"label": "complex-release-orchestrator",
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.53,
"duration_ms": 28.19,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.53,
"duration_ms": 26.4,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.52,
"duration_ms": 27.25,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.95,
"duration_ms": 29.37,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.22,
"duration_ms": 29.61,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.17,
"duration_ms": 26.32,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.13,
"duration_ms": 25.92,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.95,
"duration_ms": 26.18,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.55,
"duration_ms": 26.51,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.76,
"duration_ms": 26.36,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 28.53 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.53 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.52 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.95 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 30.22 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.17 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.13 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.95 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.55 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.76 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 28.19 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 26.4 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.25 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.37 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 29.61 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.32 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.92 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.18 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.51 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.36 | 98 | 100.0 | pass |
## Next Fixes
reports/package_verification.json
+1 -1
View File
@@ -8,7 +8,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903",
"archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3",
"archive_entry_count": 623,
"failure_count": 0,
"warning_count": 0
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903`
- Archive SHA256: `bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3`
- Failures: `0`
- Warnings: `0`
reports/registry_audit.json
+4 -4
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d",
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903"
"package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c",
"archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3"
},
"compatibility": {
"openai": "pass",
@@ -53,7 +53,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903",
"archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
"package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d`
- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903`
- Package SHA256: `f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c`
- Archive SHA256: `bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3`
- Install simulated: `True`
## Compatibility
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 200,
"package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
"package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `200`
- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d`
- Package SHA256: `f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c`
## Failures
reports/skill_atlas.json
+10 -4
View File
@@ -44,7 +44,7 @@
},
"catalog": {
"workspace_root": ".",
"generated_at": "2026-06-13",
"generated_at": "2026-06-16",
"skills": [
{
"name": "yao-meta-skill",
@@ -65,6 +65,7 @@
],
"resources": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
"scripts/build_confusion_matrix.py",
"scripts/build_skill_atlas.py",
"scripts/check_update.py",
@@ -76,6 +77,8 @@
"scripts/cross_packager.py",
"scripts/diff_eval.py",
"scripts/emit_telemetry_event.py",
"scripts/evidence_consistency_release.py",
"scripts/evidence_consistency_world_class.py",
"scripts/export_skill_ir.py",
"scripts/github_benchmark_scan.py",
"scripts/governance_check.py",
@@ -129,6 +132,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_action_evidence.py",
@@ -167,6 +171,8 @@
"scripts/yao_cli_adaptation_commands.py",
"scripts/yao_cli_config.py",
"scripts/yao_cli_create_commands.py",
"scripts/yao_cli_distribution_commands.py",
"scripts/yao_cli_output_commands.py",
"scripts/yao_cli_parser.py",
"scripts/yao_cli_report_commands.py",
"scripts/yao_cli_runtime.py",
@@ -236,8 +242,8 @@
"report": "reports/adoption_drift_report.json",
"risk_band": "low",
"event_count": 1,
"adoption_sample_count": 1,
"adoption_rate": 100.0,
"adoption_sample_count": 0,
"adoption_rate": 0,
"candidate_count": 0
}
},
@@ -900,7 +906,7 @@
"name": "incident-command-governor",
"path": "examples/governed-incident-command/generated-skill",
"reason": "review overdue by cadence monthly",
"age_days": 74,
"age_days": 77,
"allowed_days": 31,
"actionable": false,
"scope": "example"
reports/skill_os2_audit.json
+4 -4
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-15",
"generated_at": "2026-06-16",
"skill_dir": ".",
"summary": {
"item_count": 15,
@@ -145,7 +145,7 @@
"key": "benchmark-reproducibility",
"label": "Benchmark Reproducibility",
"status": "pass",
"current": "artifacts 24; missing 0; failures 3",
"current": "artifacts 25; missing 0; failures 3",
"target": "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable",
"evidence": [
{
@@ -197,7 +197,7 @@
"key": "trust-security",
"label": "Trust Security",
"status": "pass",
"current": "secrets 0; scripts 110; help failures 0",
"current": "secrets 0; scripts 113; help failures 0",
"target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable",
"evidence": [
{
@@ -285,7 +285,7 @@
"key": "registry-distribution",
"label": "Registry Distribution",
"status": "pass",
"current": "zip entries 613; install failures 0; permission failures 0",
"current": "zip entries 623; install failures 0; permission failures 0",
"target": "Package metadata, archive checksum, package verification, and install simulation pass",
"evidence": [
{
reports/skill_os2_audit.md
+4 -4
View File
@@ -1,6 +1,6 @@
# Skill OS 2.0 Audit
Generated at: `2026-06-15`
Generated at: `2026-06-16`
## Summary
@@ -21,13 +21,13 @@ Generated at: `2026-06-15`
| Output Eval Lab | pass | 5 cases; delta 100.0; exec 10; blind 5 | with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy | Add more real-file and adversarial holdout cases as usage grows. |
| Provider Holdout | external-required | model-executed 0; token-observed 0 | At least one real provider-backed holdout run with observed model/timing/token metadata | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. |
| Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. |
| Benchmark Reproducibility | pass | artifacts 24; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
| Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
| Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. |
| Trust Security | pass | secrets 0; scripts 110; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Trust Security | pass | secrets 0; scripts 113; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. |
| Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. |
| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. |
| Registry Distribution | pass | zip entries 613; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
| Registry Distribution | pass | zip entries 623; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
| Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. |
| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. |
| Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
reports/skill_os2_coverage.json
+2 -2
View File
@@ -245,7 +245,7 @@
"label": "Review Studio",
"status": "pass",
"objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.",
"current": "16 gates; decision review; warnings 4",
"current": "16 gates; decision review; warnings 3",
"command": "python3 scripts/yao.py review-studio .",
"test": "python3 tests/verify_review_studio.py",
"evidence": [
@@ -613,7 +613,7 @@
"label": "Evidence Consistency",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "31 consistency checks",
"current": "33 consistency checks",
"command": "make ci-test",
"test": "tests/verify_evidence_consistency.py",
"evidence": [
reports/skill_os2_coverage.md
+2 -2
View File
@@ -29,7 +29,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
| Trust Security | `pass` | 113 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` |
| Registry Distribution | `pass` | archive entries 623; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
| Review Studio | `pass` | 16 gates; decision review; warnings 4 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
| Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` |
## Recommended PR Coverage
@@ -48,7 +48,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
| Registry Package Format | `pass` | registry ok True | `make ci-test` | `tests/verify_registry_audit.py` |
| Review Studio 2.0 | `pass` | 16 review gates | `make ci-test` | `tests/verify_review_studio.py` |
| Migration V2 Docs | `pass` | migration guide present | `make ci-test` | `docs review` |
| Evidence Consistency | `pass` | 31 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` |
| Evidence Consistency | `pass` | 33 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` |
## Reference Extension Tracks
reports/upgrade_check.json
+2 -2
View File
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903"
"to": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d"
"to": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c"
}
]
},
scripts/evidence_consistency_release.py
+3
View File
@@ -10,6 +10,7 @@ CLEAN_LOCK_HEADER = "For final release evidence"
CLEAN_LOCK_END = "If `reports/benchmark_reproducibility.json`"
SOURCE_REFRESH_REPORT_COMMANDS = [
'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"',
'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"',
"python3 scripts/render_skill_overview.py .",
"python3 scripts/render_skill_interpretation.py .",
@@ -61,6 +62,8 @@ def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]:
"actual": actual,
"paths": [
"AGENTS.md",
"reports/context_budget.json",
"reports/context_budget_summary.json",
"reports/benchmark_reproducibility.json",
"reports/skill-overview.json",
"reports/skill-interpretation.json",
scripts/render_evidence_consistency.py
+52
View File
@@ -222,6 +222,16 @@ def compare_summary_keys(
)
def gate_by_key(review_studio: dict[str, Any], key: str) -> dict[str, Any]:
gates = review_studio.get("gates")
if not isinstance(gates, list):
return {}
for item in gates:
if isinstance(item, dict) and item.get("key") == key:
return item
return {}
def report_contract(payload: dict[str, Any]) -> dict[str, Any]:
contract = payload.get("report_contract")
return contract if isinstance(contract, dict) else {}
@@ -283,6 +293,48 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
context_stats = nested(context_budget, ["stats"], {})
claim_guard_summary = nested(claim_guard, ["summary"], {})
preflight_summary = nested(world_class_preflight, ["summary"], {})
context_governance = (
context_stats.get("deferred_resource_governance", {}) if isinstance(context_stats, dict) else {}
)
if not isinstance(context_governance, dict):
context_governance = {}
large_deferred_dirs = context_stats.get("large_deferred_resource_dirs", []) if isinstance(context_stats, dict) else []
top_deferred = "none"
if isinstance(large_deferred_dirs, list) and large_deferred_dirs:
first = large_deferred_dirs[0] if isinstance(large_deferred_dirs[0], dict) else {}
top_deferred = f"{first.get('path', 'resource')} {first.get('estimated_tokens', 'n/a')}"
context_expected_status = "pass" if context_budget.get("ok") else "block"
if context_budget.get("warnings") and context_expected_status == "pass":
context_expected_status = "warn"
context_expected_detail = (
f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/"
f"{context_stats.get('context_budget_limit', 'n/a')}; "
f"deferred {context_stats.get('deferred_resource_tokens', 'n/a')}/"
f"{context_stats.get('deferred_resource_warn_threshold', 'n/a')}; "
f"top deferred {top_deferred}; "
f"resource governance {context_governance.get('status', 'unknown')}; "
f"quality density {context_stats.get('quality_density', 'n/a')}"
)
context_gate = gate_by_key(review_studio, "context-budget")
compare_values(
checks,
key="review-studio-context-budget-mirror",
label="Review Studio mirrors context budget governance",
expected={
"status": context_expected_status,
"detail": context_expected_detail,
"evidence": REQUIRED_REPORTS["context_budget"],
},
actual={
"status": context_gate.get("status"),
"detail": context_gate.get("detail"),
"evidence": context_gate.get("evidence"),
},
paths=[REQUIRED_REPORTS["context_budget"], REQUIRED_REPORTS["review_studio"]],
detail=(
"Review Studio must not keep stale context warnings after context reports prove large deferred resources are governed."
),
)
if isinstance(benchmark_summary, dict):
compare_values(
checks,
skill_atlas/catalog.json
+9 -3
View File
@@ -1,6 +1,6 @@
{
"workspace_root": ".",
"generated_at": "2026-06-13",
"generated_at": "2026-06-16",
"skills": [
{
"name": "yao-meta-skill",
@@ -21,6 +21,7 @@
],
"resources": [
"scripts/adjudicate_output_review.py",
"scripts/apply_adaptation.py",
"scripts/build_confusion_matrix.py",
"scripts/build_skill_atlas.py",
"scripts/check_update.py",
@@ -32,6 +33,8 @@
"scripts/cross_packager.py",
"scripts/diff_eval.py",
"scripts/emit_telemetry_event.py",
"scripts/evidence_consistency_release.py",
"scripts/evidence_consistency_world_class.py",
"scripts/export_skill_ir.py",
"scripts/github_benchmark_scan.py",
"scripts/governance_check.py",
@@ -85,6 +88,7 @@
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/render_world_class_operator_runbook.py",
"scripts/render_world_class_preflight.py",
"scripts/render_world_class_submission_review.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_action_evidence.py",
@@ -123,6 +127,8 @@
"scripts/yao_cli_adaptation_commands.py",
"scripts/yao_cli_config.py",
"scripts/yao_cli_create_commands.py",
"scripts/yao_cli_distribution_commands.py",
"scripts/yao_cli_output_commands.py",
"scripts/yao_cli_parser.py",
"scripts/yao_cli_report_commands.py",
"scripts/yao_cli_runtime.py",
@@ -192,8 +198,8 @@
"report": "reports/adoption_drift_report.json",
"risk_band": "low",
"event_count": 1,
"adoption_sample_count": 1,
"adoption_rate": 100.0,
"adoption_sample_count": 0,
"adoption_rate": 0,
"candidate_count": 0
}
},
skill_atlas/stale_skills.json
+1 -1
View File
@@ -24,7 +24,7 @@
"name": "incident-command-governor",
"path": "examples/governed-incident-command/generated-skill",
"reason": "review overdue by cadence monthly",
"age_days": 74,
"age_days": 77,
"allowed_days": 31,
"actionable": false,
"scope": "example"
tests/verify_evidence_consistency.py
+38 -1
View File
@@ -49,6 +49,7 @@ def copy_reports(dst: Path) -> None:
def refresh_embedded_reports() -> None:
script_names = [
"render_context_reports.py",
"render_benchmark_reproducibility.py",
"render_skill_os2_coverage.py",
"render_world_class_evidence_plan.py",
@@ -60,10 +61,15 @@ def refresh_embedded_reports() -> None:
"render_world_class_claim_guard.py",
"render_skill_overview.py",
"render_skill_interpretation.py",
"render_review_viewer.py",
"render_review_studio.py",
]
for script_name in script_names:
command = [sys.executable, str(ROOT / "scripts" / script_name)]
if script_name != "render_context_reports.py":
command.append(str(ROOT))
subprocess.run(
[sys.executable, str(ROOT / "scripts" / script_name), str(ROOT)],
command,
cwd=ROOT,
check=True,
capture_output=True,
@@ -109,6 +115,7 @@ def assert_release_evidence_instructions_cover_first_class_reports() -> None:
source_refresh = agents_text.split(source_refresh_header, 1)[1].split(clean_lock_header, 1)[0]
clean_lock = agents_text.split(clean_lock_header, 1)[1].split("If `reports/benchmark_reproducibility.json`", 1)[0]
for block in [source_refresh, clean_lock]:
assert "python3 scripts/render_context_reports.py --generated-at \"$GENERATED_AT\"" in block, block
assert benchmark_command in block, block
assert "python3 scripts/render_skill_interpretation.py ." in block, block
assert "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"" in block, block
@@ -147,6 +154,9 @@ def main() -> None:
assert checks["release-evidence-flow-covers-first-class-reports"]["status"] == "pass", checks[
"release-evidence-flow-covers-first-class-reports"
]
assert checks["review-studio-context-budget-mirror"]["status"] == "pass", checks[
"review-studio-context-budget-mirror"
]
assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"]
assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"]
assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"]
@@ -306,6 +316,33 @@ def main() -> None:
release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]
)
context_drift_root = TMP / "context-drift-skill"
copy_reports(context_drift_root)
context_path = context_drift_root / "reports" / "context_budget.json"
context_payload = json.loads(context_path.read_text(encoding="utf-8"))
context_payload["warnings"] = ["Deferred resource footprint is high: stale test warning."]
context_payload["stats"]["deferred_resource_governance"]["status"] = "needs-review"
context_path.write_text(json.dumps(context_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
context_drift_proc = run(
[
sys.executable,
str(SCRIPT),
str(context_drift_root),
"--output-json",
str(TMP / "context_drift.json"),
"--output-md",
str(TMP / "context_drift.md"),
"--generated-at",
"2026-06-15",
]
)
assert context_drift_proc.returncode == 2, context_drift_proc.stdout
context_drift_payload = json.loads(context_drift_proc.stdout)
context_drift_checks = {item["key"]: item for item in context_drift_payload["checks"]}
assert context_drift_checks["review-studio-context-budget-mirror"]["status"] == "fail", (
context_drift_checks["review-studio-context-budget-mirror"]
)
benchmark_flow_drift_root = TMP / "benchmark-flow-drift-skill"
copy_reports(benchmark_flow_drift_root)
agents_path = benchmark_flow_drift_root / "AGENTS.md"