diff --git a/AGENTS.md b/AGENTS.md index 223cb817..1fedfd40 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,7 @@ python3 scripts/render_world_class_submission_review.py . --generated-at "$GENER python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT" python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT" +python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT" python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_overview.py . python3 scripts/render_skill_interpretation.py . @@ -60,6 +61,7 @@ python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT" For final release evidence, commit source and generated package evidence first, then run the clean-lock reports from a clean worktree: ```bash +python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT" python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_overview.py . python3 scripts/render_skill_interpretation.py . diff --git a/registry/index.json b/registry/index.json index 31ea0b4a..5af60f22 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" + "package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index f407a866..f2031ea9 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d", - "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903" + "package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c", + "archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903", + "archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index e8e1bc12..f792e823 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -16,7 +16,7 @@ "watch_line_threshold": 720, "block_line_threshold": 1500, "largest_file_lines": 899, - "watchlist_count": 8, + "watchlist_count": 9, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" @@ -43,6 +43,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 804, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/skill_report_model.py", "lines": 801, @@ -85,13 +92,6 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, - { - "path": "scripts/render_evidence_consistency.py", - "lines": 707, - "kind": "cli-script", - "severity": "pass", - "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - }, { "path": "scripts/apply_adaptation.py", "lines": 706, @@ -129,6 +129,13 @@ "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." }, + { + "path": "scripts/render_evidence_consistency.py", + "lines": 804, + "kind": "cli-script", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, { "path": "scripts/skill_report_model.py", "lines": 801, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 2fceb410..01a2705b 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -15,7 +15,7 @@ Generated at: `2026-06-16` - command modules: `6` - largest file lines: `899` - watch threshold lines: `720` -- watchlist: `8` +- watchlist: `9` - hotspots: `0` - blockers: `0` @@ -32,6 +32,7 @@ No file-size hotspots found. | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/yao_cli_parser.py` | `821` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_layout.py` | `808` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | +| `scripts/render_evidence_consistency.py` | `804` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `scripts/skill_report_model.py` | `801` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `tests/verify_review_studio.py` | `763` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/compile_skill.py` | `734` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | @@ -45,13 +46,13 @@ No file-size hotspots found. | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | | `scripts/yao_cli_parser.py` | `821` | `internal-module` | `pass` | | `scripts/skill_report_layout.py` | `808` | `internal-module` | `pass` | +| `scripts/render_evidence_consistency.py` | `804` | `cli-script` | `pass` | | `scripts/skill_report_model.py` | `801` | `internal-module` | `pass` | | `tests/verify_review_studio.py` | `763` | `test` | `pass` | | `scripts/compile_skill.py` | `734` | `cli-script` | `pass` | | `scripts/build_skill_atlas.py` | `730` | `cli-script` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | -| `scripts/render_evidence_consistency.py` | `707` | `cli-script` | `pass` | | `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` | | `scripts/review_studio_layout.py` | `694` | `internal-module` | `pass` | diff --git a/reports/context_budget.json b/reports/context_budget.json index 9fb71409..530d7af4 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -1,22 +1,20 @@ { "ok": true, "failures": [], - "warnings": [ - "Deferred resource footprint is high: 442670 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified." - ], + "warnings": [], "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1395152, + "other_text_tokens": 1404003, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1395919, - "deferred_resource_tokens": 442670, + "estimated_total_text_tokens": 1404770, + "deferred_resource_tokens": 444818, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 393459, + "estimated_tokens": 395607, "file_count": 113 }, { @@ -33,36 +31,35 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 393459, + "estimated_tokens": 395607, "file_count": 113 } ], "deferred_resource_governance": { - "status": "needs-review", + "status": "governed", "large_dir_count": 1, - "governed_large_dir_count": 0, + "governed_large_dir_count": 1, "directories": [ { - "status": "needs-review", + "status": "governed", "evidence": [ "reports/security_trust_report.json", "reports/architecture_maintainability.json", "reports/python_compatibility.json" ], "reasons": [ + "trust report covers scripts", "architecture report has no script hotspots or blockers", "Python compatibility report has no issues" ], - "missing": [ - "trust report covers scripts" - ], + "missing": [], "path": "scripts", - "estimated_tokens": 393459, + "estimated_tokens": 395607, "file_count": 113, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], - "summary": "One or more large deferred resource directories still need explicit governance evidence." + "summary": "Large deferred resources are indexed and backed by evidence." }, "relevant_file_count": 573, "unused_resource_dirs": [], diff --git a/reports/context_budget.md b/reports/context_budget.md index 0f44c15c..2e26705a 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 960 | 767 | 442670 | `needs-review` | scripts:393459 | 135.4 | - | ok | +| root | `.` | `production` | 1000 | 960 | 767 | 444818 | `governed` | scripts:395607 | 135.4 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 2c8bec4d..69afd732 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-31", + "generated_at": "2026-06-16", "targets": [ { "label": "root", @@ -8,47 +8,44 @@ "budget_limit": 1000, "initial_tokens": 960, "skill_body_tokens": 767, - "deferred_resource_tokens": 442670, + "deferred_resource_tokens": 444818, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 393459, + "estimated_tokens": 395607, "file_count": 113 } ], "deferred_resource_governance": { - "status": "needs-review", + "status": "governed", "large_dir_count": 1, - "governed_large_dir_count": 0, + "governed_large_dir_count": 1, "directories": [ { - "status": "needs-review", + "status": "governed", "evidence": [ "reports/security_trust_report.json", "reports/architecture_maintainability.json", "reports/python_compatibility.json" ], "reasons": [ + "trust report covers scripts", "architecture report has no script hotspots or blockers", "Python compatibility report has no issues" ], - "missing": [ - "trust report covers scripts" - ], + "missing": [], "path": "scripts", - "estimated_tokens": 393459, + "estimated_tokens": 395607, "file_count": 113, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], - "summary": "One or more large deferred resource directories still need explicit governance evidence." + "summary": "Large deferred resources are indexed and backed by evidence." }, "quality_density": 135.4, "unused_resource_dirs": [], "ok": true, - "warnings": [ - "Deferred resource footprint is high: 442670 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified." - ] + "warnings": [] }, { "label": "complex-release-orchestrator", diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 64a93145..3f791de1 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.53, + "duration_ms": 28.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.53, + "duration_ms": 26.4, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.52, + "duration_ms": 27.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.95, + "duration_ms": 29.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.22, + "duration_ms": 29.61, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.17, + "duration_ms": 26.32, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.13, + "duration_ms": 25.92, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.95, + "duration_ms": 26.18, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.55, + "duration_ms": 26.51, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.76, + "duration_ms": 26.36, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 5ae5b824..033695ca 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 28.53 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 27.53 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 27.52 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.95 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 30.22 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.17 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.13 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.95 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.55 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.76 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 28.19 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.4 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 27.25 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.37 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 29.61 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.32 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.92 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.18 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.51 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.36 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index ccc7dea3..56f69339 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903", + "archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3", "archive_entry_count": 623, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index 63d85716..690ebbc1 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903` +- Archive SHA256: `bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 41e1703a..a693d76c 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d", - "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903" + "package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c", + "archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903", + "archive_sha256": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" + "package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 03330217..60e218de 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d` -- Archive SHA256: `0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903` +- Package SHA256: `f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c` +- Archive SHA256: `bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3` - Install simulated: `True` ## Compatibility diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 97c05252..1a9999a5 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 200, - "package_sha256": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" + "package_sha256": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 7ffb814a..77e5ae16 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `200` -- Package SHA256: `e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d` +- Package SHA256: `f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c` ## Failures diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 7ec4ba23..de52f6a2 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -44,7 +44,7 @@ }, "catalog": { "workspace_root": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-16", "skills": [ { "name": "yao-meta-skill", @@ -65,6 +65,7 @@ ], "resources": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -76,6 +77,8 @@ "scripts/cross_packager.py", "scripts/diff_eval.py", "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -129,6 +132,7 @@ "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_action_evidence.py", @@ -167,6 +171,8 @@ "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", "scripts/yao_cli_parser.py", "scripts/yao_cli_report_commands.py", "scripts/yao_cli_runtime.py", @@ -236,8 +242,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 1, - "adoption_rate": 100.0, + "adoption_sample_count": 0, + "adoption_rate": 0, "candidate_count": 0 } }, @@ -900,7 +906,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 74, + "age_days": 77, "allowed_days": 31, "actionable": false, "scope": "example" diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index a752643a..0316789a 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-15", + "generated_at": "2026-06-16", "skill_dir": ".", "summary": { "item_count": 15, @@ -145,7 +145,7 @@ "key": "benchmark-reproducibility", "label": "Benchmark Reproducibility", "status": "pass", - "current": "artifacts 24; missing 0; failures 3", + "current": "artifacts 25; missing 0; failures 3", "target": "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable", "evidence": [ { @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 110; help failures 0", + "current": "secrets 0; scripts 113; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 613; install failures 0; permission failures 0", + "current": "zip entries 623; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 89da79aa..9023414d 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -1,6 +1,6 @@ # Skill OS 2.0 Audit -Generated at: `2026-06-15` +Generated at: `2026-06-16` ## Summary @@ -21,13 +21,13 @@ Generated at: `2026-06-15` | Output Eval Lab | pass | 5 cases; delta 100.0; exec 10; blind 5 | with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy | Add more real-file and adversarial holdout cases as usage grows. | | Provider Holdout | external-required | model-executed 0; token-observed 0 | At least one real provider-backed holdout run with observed model/timing/token metadata | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | -| Benchmark Reproducibility | pass | artifacts 24; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | +| Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 110; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 113; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 613; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 623; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index c373c3bd..84c7d43e 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -245,7 +245,7 @@ "label": "Review Studio", "status": "pass", "objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.", - "current": "16 gates; decision review; warnings 4", + "current": "16 gates; decision review; warnings 3", "command": "python3 scripts/yao.py review-studio .", "test": "python3 tests/verify_review_studio.py", "evidence": [ @@ -613,7 +613,7 @@ "label": "Evidence Consistency", "status": "pass", "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", - "current": "31 consistency checks", + "current": "33 consistency checks", "command": "make ci-test", "test": "tests/verify_evidence_consistency.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index b4341b17..1576220d 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -29,7 +29,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Trust Security | `pass` | 113 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | | Registry Distribution | `pass` | archive entries 623; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | -| Review Studio | `pass` | 16 gates; decision review; warnings 4 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | +| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | ## Recommended PR Coverage @@ -48,7 +48,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Registry Package Format | `pass` | registry ok True | `make ci-test` | `tests/verify_registry_audit.py` | | Review Studio 2.0 | `pass` | 16 review gates | `make ci-test` | `tests/verify_review_studio.py` | | Migration V2 Docs | `pass` | migration guide present | `make ci-test` | `docs review` | -| Evidence Consistency | `pass` | 31 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | +| Evidence Consistency | `pass` | 33 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | ## Reference Extension Tracks diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 0720254f..3af74f03 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "0709752db84703b213aa24bf92dedf83761571f92dc94221ae1d420f5f62e903" + "to": "bd4764d23c759a7e51e24eae68f9df70385156f38e6d958119556ff9635a18f3" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "e44836d63020474dc3343d6325d80ea05d62c581ffe16223411e8ca36362bc6d" + "to": "f984a1c13df542406c8f446d9675ddefc1c80c1b286019387f5c541602a2a29c" } ] }, diff --git a/scripts/evidence_consistency_release.py b/scripts/evidence_consistency_release.py index a73186a7..83d0b9b5 100644 --- a/scripts/evidence_consistency_release.py +++ b/scripts/evidence_consistency_release.py @@ -10,6 +10,7 @@ CLEAN_LOCK_HEADER = "For final release evidence" CLEAN_LOCK_END = "If `reports/benchmark_reproducibility.json`" SOURCE_REFRESH_REPORT_COMMANDS = [ + 'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"', 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"', "python3 scripts/render_skill_overview.py .", "python3 scripts/render_skill_interpretation.py .", @@ -61,6 +62,8 @@ def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]: "actual": actual, "paths": [ "AGENTS.md", + "reports/context_budget.json", + "reports/context_budget_summary.json", "reports/benchmark_reproducibility.json", "reports/skill-overview.json", "reports/skill-interpretation.json", diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index ca6a893c..19c98ffa 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -222,6 +222,16 @@ def compare_summary_keys( ) +def gate_by_key(review_studio: dict[str, Any], key: str) -> dict[str, Any]: + gates = review_studio.get("gates") + if not isinstance(gates, list): + return {} + for item in gates: + if isinstance(item, dict) and item.get("key") == key: + return item + return {} + + def report_contract(payload: dict[str, Any]) -> dict[str, Any]: contract = payload.get("report_contract") return contract if isinstance(contract, dict) else {} @@ -283,6 +293,48 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: context_stats = nested(context_budget, ["stats"], {}) claim_guard_summary = nested(claim_guard, ["summary"], {}) preflight_summary = nested(world_class_preflight, ["summary"], {}) + context_governance = ( + context_stats.get("deferred_resource_governance", {}) if isinstance(context_stats, dict) else {} + ) + if not isinstance(context_governance, dict): + context_governance = {} + large_deferred_dirs = context_stats.get("large_deferred_resource_dirs", []) if isinstance(context_stats, dict) else [] + top_deferred = "none" + if isinstance(large_deferred_dirs, list) and large_deferred_dirs: + first = large_deferred_dirs[0] if isinstance(large_deferred_dirs[0], dict) else {} + top_deferred = f"{first.get('path', 'resource')} {first.get('estimated_tokens', 'n/a')}" + context_expected_status = "pass" if context_budget.get("ok") else "block" + if context_budget.get("warnings") and context_expected_status == "pass": + context_expected_status = "warn" + context_expected_detail = ( + f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/" + f"{context_stats.get('context_budget_limit', 'n/a')}; " + f"deferred {context_stats.get('deferred_resource_tokens', 'n/a')}/" + f"{context_stats.get('deferred_resource_warn_threshold', 'n/a')}; " + f"top deferred {top_deferred}; " + f"resource governance {context_governance.get('status', 'unknown')}; " + f"quality density {context_stats.get('quality_density', 'n/a')}" + ) + context_gate = gate_by_key(review_studio, "context-budget") + compare_values( + checks, + key="review-studio-context-budget-mirror", + label="Review Studio mirrors context budget governance", + expected={ + "status": context_expected_status, + "detail": context_expected_detail, + "evidence": REQUIRED_REPORTS["context_budget"], + }, + actual={ + "status": context_gate.get("status"), + "detail": context_gate.get("detail"), + "evidence": context_gate.get("evidence"), + }, + paths=[REQUIRED_REPORTS["context_budget"], REQUIRED_REPORTS["review_studio"]], + detail=( + "Review Studio must not keep stale context warnings after context reports prove large deferred resources are governed." + ), + ) if isinstance(benchmark_summary, dict): compare_values( checks, diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 5406929d..e6cb4627 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -1,6 +1,6 @@ { "workspace_root": ".", - "generated_at": "2026-06-13", + "generated_at": "2026-06-16", "skills": [ { "name": "yao-meta-skill", @@ -21,6 +21,7 @@ ], "resources": [ "scripts/adjudicate_output_review.py", + "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", @@ -32,6 +33,8 @@ "scripts/cross_packager.py", "scripts/diff_eval.py", "scripts/emit_telemetry_event.py", + "scripts/evidence_consistency_release.py", + "scripts/evidence_consistency_world_class.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -85,6 +88,7 @@ "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/render_world_class_operator_runbook.py", + "scripts/render_world_class_preflight.py", "scripts/render_world_class_submission_review.py", "scripts/resource_boundary_check.py", "scripts/review_studio_action_evidence.py", @@ -123,6 +127,8 @@ "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", "scripts/yao_cli_parser.py", "scripts/yao_cli_report_commands.py", "scripts/yao_cli_runtime.py", @@ -192,8 +198,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 1, - "adoption_rate": 100.0, + "adoption_sample_count": 0, + "adoption_rate": 0, "candidate_count": 0 } }, diff --git a/skill_atlas/stale_skills.json b/skill_atlas/stale_skills.json index 6602e829..b57d361d 100644 --- a/skill_atlas/stale_skills.json +++ b/skill_atlas/stale_skills.json @@ -24,7 +24,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 74, + "age_days": 77, "allowed_days": 31, "actionable": false, "scope": "example" diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index 9f518618..dbfa9d13 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -49,6 +49,7 @@ def copy_reports(dst: Path) -> None: def refresh_embedded_reports() -> None: script_names = [ + "render_context_reports.py", "render_benchmark_reproducibility.py", "render_skill_os2_coverage.py", "render_world_class_evidence_plan.py", @@ -60,10 +61,15 @@ def refresh_embedded_reports() -> None: "render_world_class_claim_guard.py", "render_skill_overview.py", "render_skill_interpretation.py", + "render_review_viewer.py", + "render_review_studio.py", ] for script_name in script_names: + command = [sys.executable, str(ROOT / "scripts" / script_name)] + if script_name != "render_context_reports.py": + command.append(str(ROOT)) subprocess.run( - [sys.executable, str(ROOT / "scripts" / script_name), str(ROOT)], + command, cwd=ROOT, check=True, capture_output=True, @@ -109,6 +115,7 @@ def assert_release_evidence_instructions_cover_first_class_reports() -> None: source_refresh = agents_text.split(source_refresh_header, 1)[1].split(clean_lock_header, 1)[0] clean_lock = agents_text.split(clean_lock_header, 1)[1].split("If `reports/benchmark_reproducibility.json`", 1)[0] for block in [source_refresh, clean_lock]: + assert "python3 scripts/render_context_reports.py --generated-at \"$GENERATED_AT\"" in block, block assert benchmark_command in block, block assert "python3 scripts/render_skill_interpretation.py ." in block, block assert "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"" in block, block @@ -147,6 +154,9 @@ def main() -> None: assert checks["release-evidence-flow-covers-first-class-reports"]["status"] == "pass", checks[ "release-evidence-flow-covers-first-class-reports" ] + assert checks["review-studio-context-budget-mirror"]["status"] == "pass", checks[ + "review-studio-context-budget-mirror" + ] assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"] assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"] assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"] @@ -306,6 +316,33 @@ def main() -> None: release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"] ) + context_drift_root = TMP / "context-drift-skill" + copy_reports(context_drift_root) + context_path = context_drift_root / "reports" / "context_budget.json" + context_payload = json.loads(context_path.read_text(encoding="utf-8")) + context_payload["warnings"] = ["Deferred resource footprint is high: stale test warning."] + context_payload["stats"]["deferred_resource_governance"]["status"] = "needs-review" + context_path.write_text(json.dumps(context_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + context_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(context_drift_root), + "--output-json", + str(TMP / "context_drift.json"), + "--output-md", + str(TMP / "context_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert context_drift_proc.returncode == 2, context_drift_proc.stdout + context_drift_payload = json.loads(context_drift_proc.stdout) + context_drift_checks = {item["key"]: item for item in context_drift_payload["checks"]} + assert context_drift_checks["review-studio-context-budget-mirror"]["status"] == "fail", ( + context_drift_checks["review-studio-context-budget-mirror"] + ) + benchmark_flow_drift_root = TMP / "benchmark-flow-drift-skill" copy_reports(benchmark_flow_drift_root) agents_path = benchmark_flow_drift_root / "AGENTS.md"