From e2933ce516bad14a403fa5cb58c385749b441572 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Sat, 13 Jun 2026 20:29:01 +0800 Subject: [PATCH] feat: surface telemetry drift in skill atlas --- README.md | 3 +- docs/migration-v2.md | 2 + references/skill-atlas-method.md | 9 +- references/telemetry-drift-method.md | 5 + registry/index.json | 2 +- registry/packages/yao-meta-skill.json | 6 +- reports/adoption_drift_report.json | 2 +- reports/context_budget.json | 6 +- reports/install_simulation.json | 2 +- reports/output_execution_runs.json | 20 +- reports/output_execution_runs.md | 20 +- reports/package_verification.json | 4 +- reports/package_verification.md | 2 +- reports/registry_audit.json | 8 +- reports/registry_audit.md | 4 +- reports/review-studio.html | 18 +- reports/review-studio.json | 256 +++++++++++++++++++------- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/skill-os-2-review.md | 24 +-- reports/skill-overview.html | 8 +- reports/skill-overview.json | 35 ++-- reports/skill_atlas.html | 5 +- reports/skill_atlas.json | 141 ++++++++++++-- reports/upgrade_check.json | 4 +- scripts/build_skill_atlas.py | 136 +++++++++++++- scripts/render_review_studio.py | 3 + scripts/review_studio_gates.py | 4 +- skill_atlas/catalog.json | 135 ++++++++++++-- skill_atlas/drift_signals.json | 1 + tests/verify_review_studio.py | 2 + tests/verify_skill_atlas.py | 56 +++++- 32 files changed, 739 insertions(+), 188 deletions(-) create mode 100644 skill_atlas/drift_signals.json diff --git a/README.md b/README.md index bdda82f..3e5588c 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - three high-value next iteration directions after the first package is created - a lightweight feedback log that does not require a full promotion cycle - a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates +- a Skill Atlas drift layer that reads aggregate adoption reports and surfaces portfolio-level drift signals without packaging raw telemetry logs - a baseline compare report for with-skill vs baseline review - a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits - Skill IR as the platform-neutral semantic contract, plus compiler reports and client-specific adapters @@ -406,7 +407,7 @@ Utility scripts that make the meta-skill operational: - `render_review_annotations.py`: records reviewer annotations tied to Review Studio gates, source/report paths, and optional line numbers, with open blocker annotations reflected in Review Studio decisions - `run_conformance_suite.py`: verifies runtime conformance for OpenAI, Claude, Agent Skills, VS Code/Copilot-style, and generic targets - `trust_check.py`: generates the trust/security report for scripts, dependencies, secret risk, bounded network host policy, execution-level `--help` smoke checks, permission inputs, trust metadata, and stable source-contract integrity -- `build_skill_atlas.py`: builds the Skill Atlas catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview for a multi-skill workspace +- `build_skill_atlas.py`: builds the Skill Atlas catalog, route-overlap matrix, dependency graph, stale report, owner gaps, aggregate drift signals, and HTML overview for a multi-skill workspace - `registry_audit.py`: builds registry package metadata and audits version, owner, license, checksum, Skill IR source, and compatibility matrix - `verify_package.py`: verifies generated package manifests, target adapters, zip archive safety, archive checksum, and registry parity - `simulate_install.py`: extracts a generated zip into a temporary skill root and verifies entrypoint, manifest, interface, reports, and adapters can be loaded diff --git a/docs/migration-v2.md b/docs/migration-v2.md index 0b4a349..7189997 100644 --- a/docs/migration-v2.md +++ b/docs/migration-v2.md @@ -41,6 +41,8 @@ This document tracks the first migration path from Yao Meta Skill 1.x to the Ski python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json ``` + This also writes `skill_atlas/drift_signals.json` from aggregate `reports/adoption_drift_report.json` files. Do not migrate or publish raw `reports/telemetry_events.jsonl` logs. + 5. Keep legacy trigger eval gates intact while output eval coverage grows. 6. Move adapter-specific logic toward future compiler commands after IR fields are stable. diff --git a/references/skill-atlas-method.md b/references/skill-atlas-method.md index a0f0455..9c58043 100644 --- a/references/skill-atlas-method.md +++ b/references/skill-atlas-method.md @@ -16,6 +16,7 @@ Single-skill quality is not enough for a team library. A skill portfolio also ne - Flag missing owner or review metadata. - Flag stale skills based on `updated_at` and `review_cadence`. - Extract no-route opportunities from failure notes. +- Read aggregate adoption drift reports and flag telemetry drift without reading raw telemetry logs. ## Scope Policy @@ -23,6 +24,12 @@ Atlas keeps a full catalog, but release gates should distinguish actionable libr Use `skill_atlas/policy.json` to mark path prefixes as non-actionable when they are intentionally retained as examples, evolution snapshots, embedded generated skills, or validator fixtures. Non-actionable items still appear in the full report, route matrix, stale list, and owner gap list, but Review Studio should use the actionable counts for release readiness. +## Telemetry Link + +Atlas may read each skill's aggregate `reports/adoption_drift_report.json` to surface portfolio drift signals such as no telemetry for production/library/governed skills, missed triggers, bad outputs, missing resources, script errors, and review-overdue counts. It must not read or package `reports/telemetry_events.jsonl`; raw telemetry remains local-only evidence owned by the skill. + +Write drift output to `skill_atlas/drift_signals.json`. Non-actionable scopes stay visible in that file and in the HTML report, but only actionable drift signals should affect release readiness. + ## Reviewer Gate -Use Atlas before promoting a single skill into a shared library. If an actionable route collision, missing owner, or stale governed skill appears, fix the portfolio boundary before adding more local complexity to one skill. Non-actionable issues should stay visible as evidence, not as release blockers. +Use Atlas before promoting a single skill into a shared library. If an actionable route collision, missing owner, stale governed skill, or telemetry drift signal appears, fix the portfolio boundary before adding more local complexity to one skill. Non-actionable issues should stay visible as evidence, not as release blockers. diff --git a/references/telemetry-drift-method.md b/references/telemetry-drift-method.md index 8633108..071bbb2 100644 --- a/references/telemetry-drift-method.md +++ b/references/telemetry-drift-method.md @@ -54,7 +54,12 @@ Package builders should exclude `reports/telemetry_events.jsonl`. The root repos 4. Convert bad outputs into Output Eval assertions and failure taxonomy entries. 5. Convert script errors into non-interactive smoke tests. 6. Feed review-overdue signals back into Skill Atlas and owner review. +7. Let Skill Atlas read only `reports/adoption_drift_report.json` and publish portfolio-level `skill_atlas/drift_signals.json`. ## Review Studio Role Review Studio should show the aggregate telemetry gate as an operating loop, not as raw logs. A blocker means the telemetry contract was violated. A warning means the evidence is absent or the drift signal needs a follow-up case. + +## Skill Atlas Role + +Skill Atlas uses aggregate adoption drift reports to rank portfolio work. It should surface no-data warnings for actionable production/library/governed skills, and drift warnings for missed triggers, wrong triggers, bad outputs, missing resources, script errors, and review-overdue counts. It must not inspect raw JSONL telemetry or use non-actionable example/fixture signals as release blockers. diff --git a/registry/index.json b/registry/index.json index 95d8dac..052a2d4 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 17cd7b3..b097001 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3", - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 4f92a9f..d2a13ef 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T12:15:07Z", + "generated_at": "2026-06-13T12:26:25Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/context_budget.json b/reports/context_budget.json index 0f17dd4..f9c3c9a 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 905782, + "other_text_tokens": 910665, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 906533, - "relevant_file_count": 386, + "estimated_total_text_tokens": 911416, + "relevant_file_count": 388, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index d24dfa8..5707372 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 493, + "archive_entry_count": 494, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index b2a6fec..33a74b8 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.86, + "duration_ms": 31.8, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.72, + "duration_ms": 31.63, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.59, + "duration_ms": 30.68, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.19, + "duration_ms": 32.69, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.03, + "duration_ms": 32.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.71, + "duration_ms": 34.03, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.23, + "duration_ms": 34.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.55, + "duration_ms": 34.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.21, + "duration_ms": 33.14, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.89, + "duration_ms": 36.79, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 4261e1e..ad89098 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 28.86 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 28.72 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 29.59 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 30.19 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 31.03 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 29.71 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 32.23 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 31.55 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 32.21 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 31.89 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 31.8 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 31.63 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 30.68 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 32.69 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 32.73 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 34.03 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 34.95 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 34.6 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 33.14 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 36.79 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 5140cd1..4c9e15d 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", - "archive_entry_count": 493, + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index fec4655..e4ab673 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060` +- Archive SHA256: `23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 0d976ec..e731b25 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3", - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index ab68f06..89523c5 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3` -- Archive SHA256: `176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060` +- Package SHA256: `ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb` +- Archive SHA256: `23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index a8493c0..c7d40e8 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -242,12 +242,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

69 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

493 zip entries; package verification

Installpass

4 adapters readable; local install simulation

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

69 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

494 zip entries; package verification

Installpass

4 adapters readable; local install simulation

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 69 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 69 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
@@ -279,7 +279,7 @@

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

-

组合治理

Skill 数
12
纳入审查
1
待处理冲突
0
待处理 owner
0
待处理过期
0
非行动项
24
+

组合治理

Skill 数
12
纳入审查
1
待处理冲突
0
待处理 owner
0
待处理过期
0
Telemetry Report Count
1
Actionable Drift Signal Count
0
Drift Signal Count
0
非行动项
24
@@ -313,7 +313,7 @@
-

信任报告

Secret
0
脚本数
69
网络脚本
3
Help 失败
0
包体哈希
2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd
+

信任报告

Secret
0
脚本数
69
网络脚本
3
Help 失败
0
包体哈希
ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -328,13 +328,13 @@
-

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

+

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

下一动作

优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再用运营回路判断真实影响。

-

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

-

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

+

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
@@ -344,12 +344,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
493
失败数
0
警告数
0
归档哈希
176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
494
失败数
0
警告数
0
归档哈希
23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20
diff --git a/reports/review-studio.json b/reports/review-studio.json index 9609b62..2e98744 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -83,7 +83,7 @@ "key": "skill-atlas", "label": "组合治理", "status": "pass", - "detail": "12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues", + "detail": "12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues", "evidence": "reports/skill_atlas.json", "link": "skill_atlas.html" }, @@ -91,7 +91,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low", + "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low", "evidence": "reports/adoption_drift_report.json", "link": "adoption_drift_report.md" }, @@ -355,7 +355,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 350 个词/字，references 约 14450 个词/字。", + "入口约 350 个词/字，references 约 14610 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -459,7 +459,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 350 个词/字，references 约 14450 个词/字。" + "上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -1226,7 +1226,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" }, "skill_atlas": { "skill_count": 12, @@ -1239,6 +1239,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "registry_distribution": { @@ -1261,8 +1264,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd", - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, "compatibility": { "openai": "pass", @@ -1293,7 +1296,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1309,8 +1312,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", - "archive_entry_count": 493, + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 }, @@ -1321,7 +1324,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 493, + "archive_entry_count": 494, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1384,12 +1387,12 @@ { "field": "archive_sha256", "from": "", - "to": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd" + "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" } ] }, @@ -1406,14 +1409,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1422,7 +1425,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {} }, @@ -3119,7 +3122,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.87, + "duration_ms": 33.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3147,7 +3150,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.27, + "duration_ms": 31.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3170,7 +3173,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.04, + "duration_ms": 30.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3198,7 +3201,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 41.0, + "duration_ms": 29.84, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3221,7 +3224,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 37.68, + "duration_ms": 29.53, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3249,7 +3252,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 37.33, + "duration_ms": 28.84, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3272,7 +3275,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.73, + "duration_ms": 29.54, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3299,7 +3302,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.69, + "duration_ms": 29.69, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3322,7 +3325,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.1, + "duration_ms": 29.54, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3351,7 +3354,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.27, + "duration_ms": 28.91, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -8212,7 +8215,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" }, "failures": [], "warnings": [], @@ -10041,10 +10044,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 893597, + "other_text_tokens": 910655, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 894348, - "relevant_file_count": 386, + "estimated_total_text_tokens": 911406, + "relevant_file_count": 388, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -10856,6 +10859,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "scope_policy": { @@ -11031,7 +11037,16 @@ ], "atlas_scope": "release", "actionable": true, - "scope_reason": "default release-actionable skill" + "scope_reason": "default release-actionable skill", + "telemetry": { + "report_present": true, + "report": "reports/adoption_drift_report.json", + "risk_band": "low", + "event_count": 1, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "release-orchestrator", @@ -11089,7 +11104,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/complex-release-orchestrator/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -11128,7 +11152,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/final/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -11151,7 +11184,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v0/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -11189,7 +11231,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v1/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "incident-command-governor", @@ -11230,7 +11281,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/governed-incident-command/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "note-cleanup", @@ -11267,7 +11327,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/simple-note-cleanup/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -11307,7 +11376,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/team-frontend-review/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "geo-ranking-article-generator", @@ -11367,7 +11445,16 @@ ], "atlas_scope": "embedded-generated-skill", "actionable": false, - "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release.", + "telemetry": { + "report_present": false, + "report": "geo-ranking-article-generator/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "invalid-governance-skill", @@ -11389,7 +11476,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/governance_invalid_manifest/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-yaml-skill", @@ -11413,7 +11509,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_invalid_yaml/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-skill", @@ -11436,7 +11541,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_missing_interface_field/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } } ], "summary": { @@ -11450,6 +11564,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 } }, @@ -11740,6 +11857,8 @@ } ], "actionable_owner_review_gaps": [], + "drift_signals": [], + "actionable_drift_signals": [], "no_route_opportunities": [], "artifacts": { "catalog": "skill_atlas/catalog.json", @@ -11747,6 +11866,7 @@ "dependency_graph": "skill_atlas/dependency_graph.json", "stale_skills": "skill_atlas/stale_skills.json", "owner_review_gaps": "skill_atlas/owner_review_gaps.json", + "drift_signals": "skill_atlas/drift_signals.json", "no_route_opportunities": "skill_atlas/no_route_opportunities.json", "report_json": "reports/skill_atlas.json", "report_html": "reports/skill_atlas.html" @@ -11755,7 +11875,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T12:10:53Z", + "generated_at": "2026-06-13T12:24:01Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -11779,14 +11899,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -11795,7 +11915,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {} }, @@ -11803,29 +11923,29 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 1, - "accepted": 1, + "adoption_events": 0, + "accepted": 0, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 100.0 + "adoption_rate": 0 } ], "next_iteration_candidates": [], "recent_events": [ { - "event": "skill_activation", + "event": "review_event", "skill": "yao-meta-skill", "version": "1.1.0", - "activation_type": "explicit", - "outcome": "accepted", + "activation_type": "manual", + "outcome": "reviewed", "failure_type": "none", - "timestamp": "2026-06-13T10:00:00Z" + "timestamp": "2026-06-13T12:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "events_jsonl": "reports/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -11874,7 +11994,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "source": "reports/review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -11917,8 +12037,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd", - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, "compatibility": { "openai": "pass", @@ -11949,7 +12069,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -11974,7 +12094,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" } ] }, @@ -11997,8 +12117,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", - "archive_entry_count": 493, + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 }, @@ -12663,7 +12783,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 493, + "archive_entry_count": 494, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -12856,12 +12976,12 @@ { "field": "archive_sha256", "from": "", - "to": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd" + "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index d9327be..aa9ccef 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 96e66cf..a9c9838 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `153` -- Package SHA256: `7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3` +- Package SHA256: `ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb` ## Failures diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index e53385e..78a3ab2 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -11,7 +11,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Output Eval Lab v0 for with-skill vs baseline assertion grading plus local command-runner execution evidence, blind A/B review pack, separate answer key evidence, and reviewer adjudication reporting. - Runtime Conformance v0 for target-consumption checks. - Trust/Security v0 for secret, dependency, script, bounded network host policy, execution-level CLI help smoke checks, trust metadata, and stable source-contract integrity checks. -- Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, and no-route opportunities. +- Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, aggregate drift signals, and no-route opportunities. - Bilingual Skill Overview v2 that includes these evidence surfaces. - Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review. - Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers. @@ -21,7 +21,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Package Verification v0 for generated manifest, target adapters, zip archive safety, archive checksum, and registry parity. - Install Simulation v0 for local extraction, entrypoint loading, interface loading, report presence, and adapter readability. - Upgrade Check v0 for registry package diffs, semver bump recommendations, breaking-change notes, and release evidence. -- Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, and iteration candidate generation. +- Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, iteration candidate generation, and Skill Atlas drift-signal input. - Review Waivers v0 for human warning acceptance with reviewer, reason, scope, expiry, and blocker-safe policy. - Governed Permission Gates v0 for reviewer-approved network, file-write, and subprocess capabilities with scope, reason, expiry, evidence, and target-enforcement mapping. - Runtime Permission Probes v0 for packaged target adapter checks, explicit native-enforcement flags, metadata fallback evidence, and residual permission risks. @@ -31,7 +31,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning. - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. -This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, and a provider-backed output runner exists. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, real client telemetry capture, installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, a provider-backed output runner exists, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, real client telemetry capture, installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open. ## Coverage Matrix @@ -46,7 +46,7 @@ This is still not the final world-class state. Target-native behavior contracts | Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed | | Review Studio Source Refs | `scripts/render_review_studio.py`, `tests/verify_review_studio.py`, `references/review-studio-method.md` | v0 landed | | Review Annotations | `scripts/render_review_annotations.py`, `reports/review_annotations.md`, `tests/verify_review_annotations.py`, Review Studio annotation panel and blocker decision hook | v0 landed | -| Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `reports/skill_atlas.html` | v0 landed | +| Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `skill_atlas/drift_signals.json`, `reports/skill_atlas.html` | v0 landed | | Registry & Distribution | `registry/*.schema.json`, `scripts/registry_audit.py`, `reports/registry_audit.md`, `registry/packages/yao-meta-skill.json` | v0 landed | | Package Verification | `scripts/verify_package.py`, `reports/package_verification.md`, `tests/verify_package_verification.py` | v0 landed | | Install Simulation | `scripts/simulate_install.py`, `reports/install_simulation.md`, `tests/verify_install_simulation.py` | v0 landed | @@ -79,11 +79,11 @@ The Review Studio page aggregates intent, trigger, output, context, runtime, tru Next move: add richer source-line anchors inside generated reports and record real reviewer annotations during the next human review pass. -### 4. Multi-skill operation now has v0 coverage, but needs real telemetry +### 4. Multi-skill operation now links Atlas with drift, but needs real client capture -The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, and feed next iteration candidates into Review Studio. It still needs real client-side capture instead of manual CLI event recording. +The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, aggregate drift signals, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, feed next iteration candidates into Review Studio, and publish aggregate drift input for Atlas. It still needs real client-side capture instead of manual CLI event recording. -Next move: connect client activations and failure history so Atlas can rank stale or conflicting skills by real usage impact. +Next move: connect client activations and failure history so Atlas can rank stale, drifting, or conflicting skills by real usage impact. ### 5. Trust report is structural, not full security review @@ -101,12 +101,12 @@ Next move: add real client or installer permission enforcement integration. | Trust | `0` secret findings, `1` pinned dependency file, `9` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `60 / 60` CLI help smoke checks passing across `69` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | -| Skill Atlas | `12` scanned skills, `1` actionable root skill, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `24` scoped non-actionable issue signals retained for visibility | +| Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `493` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `493` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `494` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `494` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `0` failures, `0` warnings | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | -| Adoption Drift | `1` metadata-only activation event, `1` adoption sample, adoption `100.0`, risk band `low`; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | +| Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | | Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0 | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | | Review Studio | decision `review`, world-class score `92`, `13` gates, `0` blockers, `2` warnings, `2` review actions, `0` open annotation blockers | @@ -120,4 +120,4 @@ Next move: add real client or installer permission enforcement integration. 2. Add native client or installer enforcement for approved high-permission capabilities. 3. Run the new provider-backed output runner against holdout cases with real credentials, then record the current blind A/B decisions before claiming fully ready status. 4. Add real reviewer annotation records during the next human review pass. -5. Connect Skill Atlas with real client telemetry and drift history. +5. Connect real client telemetry capture and failure history so Atlas drift signals are based on live usage, not manual samples. diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 44a7c30..80b1a19 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -609,7 +609,7 @@
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • Skill 作者、复用团队和后续 reviewer。Skill 作者、复用团队和后续 reviewer。
  • 创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。
-
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 350 个词/字，references 约 14450 个词/字。入口约 350 个词/字，references 约 14450 个词/字。
+
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 350 个词/字，references 约 14610 个词/字。入口约 350 个词/字，references 约 14610 个词/字。
@@ -641,10 +641,10 @@

指标判读Reading

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 350 个词/字，references 约 14450 个词/字。入口约 350 个词/字，references 约 14450 个词/字。
+
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 350 个词/字，references 约 14610 个词/字。入口约 350 个词/字，references 约 14610 个词/字。
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 350 个词/字，references 约 14450 个词/字。入口约 350 个词/字，references 约 14450 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 350 个词/字，references 约 14610 个词/字。入口约 350 个词/字，references 约 14610 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
@@ -720,7 +720,7 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 350 个词/字，references 约 14450 个词/字。上下文成本需要补强：入口约 350 个词/字，references 约 14450 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。纳入下一轮修复Fix next
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 0b68b3d..ec62842 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -96,7 +96,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 350 个词/字，references 约 14450 个词/字。", + "入口约 350 个词/字，references 约 14610 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -200,7 +200,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 350 个词/字，references 约 14450 个词/字。" + "上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -967,7 +967,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" }, "skill_atlas": { "skill_count": 12, @@ -980,6 +980,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "registry_distribution": { @@ -1002,8 +1005,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd", - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, "compatibility": { "openai": "pass", @@ -1034,7 +1037,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1050,8 +1053,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060", - "archive_entry_count": 493, + "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 }, @@ -1062,7 +1065,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 493, + "archive_entry_count": 494, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1125,12 +1128,12 @@ { "field": "archive_sha256", "from": "", - "to": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "2109ae8e212111b1619d8d728e54c73425fecc174113b45629c656ecceb3fadd" + "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" } ] }, @@ -1147,14 +1150,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 1, - "activation_count": 1, - "accepted_count": 1, + "adoption_sample_count": 0, + "activation_count": 0, + "accepted_count": 0, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 100.0, + "adoption_rate": 0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1163,7 +1166,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "skill_activation": 1 + "review_event": 1 }, "failure_types": {} }, diff --git a/reports/skill_atlas.html b/reports/skill_atlas.html index 0c14b4a..8812ffd 100644 --- a/reports/skill_atlas.html +++ b/reports/skill_atlas.html @@ -22,13 +22,14 @@

Skill Atlas

-

Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

+

Portfolio-level review for route overlap, stale ownership, shared resources, telemetry drift, and no-route opportunities.

Skills12
Actionable1
Route Collisions0
Owner Gaps0
Stale Skills0
+
Drift Signals0
No-Route Opportunities0
@@ -37,7 +38,7 @@

Full Portfolio Counts

-

All scanned skills remain visible: 5 total route collisions, 9 total owner gaps, and 10 total stale signals.

+

All scanned skills remain visible: 5 total route collisions, 9 total owner gaps, 10 total stale signals, and 0 telemetry drift signals.

Catalog

diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 249003c..c55e956 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -12,6 +12,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 }, "scope_policy": { @@ -187,7 +190,16 @@ ], "atlas_scope": "release", "actionable": true, - "scope_reason": "default release-actionable skill" + "scope_reason": "default release-actionable skill", + "telemetry": { + "report_present": true, + "report": "reports/adoption_drift_report.json", + "risk_band": "low", + "event_count": 1, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "release-orchestrator", @@ -245,7 +257,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/complex-release-orchestrator/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -284,7 +305,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/final/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -307,7 +337,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v0/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -345,7 +384,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v1/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "incident-command-governor", @@ -386,7 +434,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/governed-incident-command/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "note-cleanup", @@ -423,7 +480,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/simple-note-cleanup/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -463,7 +529,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/team-frontend-review/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "geo-ranking-article-generator", @@ -523,7 +598,16 @@ ], "atlas_scope": "embedded-generated-skill", "actionable": false, - "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release.", + "telemetry": { + "report_present": false, + "report": "geo-ranking-article-generator/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "invalid-governance-skill", @@ -545,7 +629,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/governance_invalid_manifest/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-yaml-skill", @@ -569,7 +662,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_invalid_yaml/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-skill", @@ -592,7 +694,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_missing_interface_field/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } } ], "summary": { @@ -606,6 +717,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 } }, @@ -896,6 +1010,8 @@ } ], "actionable_owner_review_gaps": [], + "drift_signals": [], + "actionable_drift_signals": [], "no_route_opportunities": [], "artifacts": { "catalog": "skill_atlas/catalog.json", @@ -903,6 +1019,7 @@ "dependency_graph": "skill_atlas/dependency_graph.json", "stale_skills": "skill_atlas/stale_skills.json", "owner_review_gaps": "skill_atlas/owner_review_gaps.json", + "drift_signals": "skill_atlas/drift_signals.json", "no_route_opportunities": "skill_atlas/no_route_opportunities.json", "report_json": "reports/skill_atlas.json", "report_html": "reports/skill_atlas.html" diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index cc8ff8f..b3043f1 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "176d93befb446985aaff7cd8325b05c4858da13177cb13db82d210aa2d264060" + "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "7a8dab0d542d2994adec2045b54559612db36f14db8b7b66e4138fb0435ae3d3" + "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" } ] }, diff --git a/scripts/build_skill_atlas.py b/scripts/build_skill_atlas.py index 4e1b3f4..d88a2b7 100644 --- a/scripts/build_skill_atlas.py +++ b/scripts/build_skill_atlas.py @@ -57,6 +57,7 @@ DEFAULT_SCOPE = { "actionable": True, "scope_reason": "default release-actionable skill", } +TELEMETRY_REQUIRED_MATURITIES = {"production", "library", "governed"} def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]: @@ -217,6 +218,105 @@ def collect_skill(workspace_root: Path, skill_dir: Path, policy: dict[str, Any]) } +def load_telemetry_profile(workspace_root: Path, skill_dir: Path, skill: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + report_path = skill_dir / "reports" / "adoption_drift_report.json" + rel_report = safe_rel(workspace_root, report_path) + maturity = str(skill.get("maturity", "")).casefold() + report_present = report_path.exists() + telemetry = { + "report_present": report_present, + "report": rel_report, + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0, + } + signals: list[dict[str, Any]] = [] + if not report_present: + if skill.get("actionable") and maturity in TELEMETRY_REQUIRED_MATURITIES: + signals.append( + { + "name": skill["name"], + "path": skill["path"], + "source": rel_report, + "risk_band": "no-data", + "signal_types": ["no telemetry"], + "recommendation": "Render adoption drift evidence or record a small metadata-only sample before release review.", + "actionable": bool(skill.get("actionable")), + "scope": str(skill.get("atlas_scope", "")), + "summary": {"event_count": 0, "adoption_sample_count": 0}, + } + ) + return telemetry, signals + + payload = load_json(report_path) + summary = payload.get("summary", {}) if isinstance(payload.get("summary"), dict) else {} + candidates = payload.get("next_iteration_candidates", []) + candidates = candidates if isinstance(candidates, list) else [] + risk_band = str(summary.get("risk_band") or "unknown") + telemetry.update( + { + "risk_band": risk_band, + "event_count": int(summary.get("event_count") or 0), + "adoption_sample_count": int(summary.get("adoption_sample_count") or 0), + "adoption_rate": summary.get("adoption_rate", 0), + "candidate_count": len(candidates), + } + ) + + signal_types: list[str] = [] + if telemetry["event_count"] == 0 and maturity in TELEMETRY_REQUIRED_MATURITIES: + signal_types.append("no telemetry") + if int(summary.get("missed_trigger_count") or 0): + signal_types.append("missed trigger") + if int(summary.get("wrong_trigger_count") or 0): + signal_types.append("wrong trigger") + if int(summary.get("bad_output_count") or 0): + signal_types.append("bad output") + if int(summary.get("missing_resource_count") or 0): + signal_types.append("missing resource") + if int(summary.get("script_error_count") or 0): + signal_types.append("script error") + if int(summary.get("review_overdue_count") or 0): + signal_types.append("review overdue") + if risk_band in {"medium", "high"} and not signal_types: + signal_types.append("telemetry drift") + + if signal_types: + recommendation = "Convert telemetry drift into eval, trust, or owner-review actions." + for candidate in candidates: + if not isinstance(candidate, dict): + continue + if str(candidate.get("signal", "")) in signal_types: + recommendation = str(candidate.get("recommendation") or recommendation) + break + signals.append( + { + "name": skill["name"], + "path": skill["path"], + "source": rel_report, + "risk_band": risk_band, + "signal_types": signal_types, + "recommendation": recommendation, + "actionable": bool(skill.get("actionable")), + "scope": str(skill.get("atlas_scope", "")), + "summary": { + "event_count": telemetry["event_count"], + "adoption_sample_count": telemetry["adoption_sample_count"], + "adoption_rate": telemetry["adoption_rate"], + "missed_trigger_count": int(summary.get("missed_trigger_count") or 0), + "wrong_trigger_count": int(summary.get("wrong_trigger_count") or 0), + "bad_output_count": int(summary.get("bad_output_count") or 0), + "missing_resource_count": int(summary.get("missing_resource_count") or 0), + "script_error_count": int(summary.get("script_error_count") or 0), + "review_overdue_count": int(summary.get("review_overdue_count") or 0), + }, + } + ) + return telemetry, signals + + def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: rows = [] collisions = [] @@ -392,9 +492,14 @@ def render_html(payload: dict[str, Any]) -> str: f"{html.escape(skill.get('atlas_scope') or 'release')}" "" ) - blockers = payload["actionable_route_collisions"][:20] + payload["actionable_owner_review_gaps"][:20] + payload["actionable_stale_skills"][:20] + blockers = ( + payload["actionable_route_collisions"][:20] + + payload["actionable_owner_review_gaps"][:20] + + payload["actionable_stale_skills"][:20] + + payload["actionable_drift_signals"][:20] + ) blocker_items = "".join( - f"
  • {html.escape(item.get('name', item.get('skill_a', 'issue')))} {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', [])))))}
    • " + f"
  • {html.escape(item.get('name', item.get('skill_a', 'issue')))} {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', item.get('signal_types', []))))))}
    • " for item in blockers ) return f""" @@ -421,13 +526,14 @@ def render_html(payload: dict[str, Any]) -> str:

    Skill Atlas

    -

    Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

    +

    Portfolio-level review for route overlap, stale ownership, shared resources, telemetry drift, and no-route opportunities.

    Skills{summary['skill_count']}
    Actionable{summary['actionable_skill_count']}
    Route Collisions{summary['actionable_route_collision_count']}
    Owner Gaps{summary['actionable_owner_gap_count']}
    Stale Skills{summary['actionable_stale_count']}
    +
    Drift Signals{summary['actionable_drift_signal_count']}
    No-Route Opportunities{summary['no_route_opportunity_count']}
    @@ -436,7 +542,7 @@ def render_html(payload: dict[str, Any]) -> str:

    Full Portfolio Counts

    -

    All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, and {summary['stale_count']} total stale signals.

    +

    All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, {summary['stale_count']} total stale signals, and {summary['drift_signal_count']} telemetry drift signals.

    Catalog

    @@ -455,7 +561,16 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor workspace_root = workspace_root.resolve() scope_policy = load_scope_policy(workspace_root) skill_dirs = find_skill_dirs(workspace_root) - skills = [collect_skill(workspace_root, skill_dir, scope_policy) for skill_dir in skill_dirs] + skills = [] + drift_signals: list[dict[str, Any]] = [] + telemetry_report_count = 0 + for skill_dir in skill_dirs: + skill = collect_skill(workspace_root, skill_dir, scope_policy) + telemetry, signals = load_telemetry_profile(workspace_root, skill_dir, skill) + skill["telemetry"] = telemetry + telemetry_report_count += 1 if telemetry["report_present"] else 0 + drift_signals.extend(signals) + skills.append(skill) overlap_rows, collisions = route_overlap(skills, threshold) graph = dependency_graph(skills) stale = stale_skills(skills, today) @@ -465,6 +580,7 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor actionable_collisions = [item for item in collisions if item.get("actionable")] actionable_stale = [item for item in stale if item.get("actionable")] actionable_owner_gaps = [item for item in owner_gaps if item.get("actionable")] + actionable_drift_signals = [item for item in drift_signals if item.get("actionable")] summary = { "skill_count": len(skills), "actionable_skill_count": len(actionable_skills), @@ -476,9 +592,13 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor "actionable_stale_count": len(actionable_stale), "shared_resource_count": len(graph["shared_resources"]), "no_route_opportunity_count": len(opportunities), + "telemetry_report_count": telemetry_report_count, + "drift_signal_count": len(drift_signals), + "actionable_drift_signal_count": len(actionable_drift_signals), "non_actionable_issue_count": (len(collisions) - len(actionable_collisions)) + (len(owner_gaps) - len(actionable_owner_gaps)) - + (len(stale) - len(actionable_stale)), + + (len(stale) - len(actionable_stale)) + + (len(drift_signals) - len(actionable_drift_signals)), } catalog = { "workspace_root": display_path(workspace_root), @@ -499,6 +619,8 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor "actionable_stale_skills": actionable_stale, "owner_review_gaps": owner_gaps, "actionable_owner_review_gaps": actionable_owner_gaps, + "drift_signals": drift_signals, + "actionable_drift_signals": actionable_drift_signals, "no_route_opportunities": opportunities, "artifacts": { "catalog": display_path(output_dir / "catalog.json"), @@ -506,6 +628,7 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor "dependency_graph": display_path(output_dir / "dependency_graph.json"), "stale_skills": display_path(output_dir / "stale_skills.json"), "owner_review_gaps": display_path(output_dir / "owner_review_gaps.json"), + "drift_signals": display_path(output_dir / "drift_signals.json"), "no_route_opportunities": display_path(output_dir / "no_route_opportunities.json"), "report_json": display_path(report_json), "report_html": display_path(report_html), @@ -517,6 +640,7 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor (output_dir / "dependency_graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (output_dir / "stale_skills.json").write_text(json.dumps(stale, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (output_dir / "owner_review_gaps.json").write_text(json.dumps(owner_gaps, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output_dir / "drift_signals.json").write_text(json.dumps(drift_signals, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") (output_dir / "no_route_opportunities.json").write_text(json.dumps(opportunities, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") report_json.parent.mkdir(parents=True, exist_ok=True) report_html.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index 989ad5f..3bd2a1b 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -634,6 +634,9 @@ def render_html(report: dict[str, Any]) -> str: "actionable_route_collision_count", "actionable_owner_gap_count", "actionable_stale_count", + "telemetry_report_count", + "actionable_drift_signal_count", + "drift_signal_count", "non_actionable_issue_count", ], "skill atlas summary missing", diff --git a/scripts/review_studio_gates.py b/scripts/review_studio_gates.py index b1609e9..88b2877 100644 --- a/scripts/review_studio_gates.py +++ b/scripts/review_studio_gates.py @@ -327,7 +327,8 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An ) actionable_owner_gaps = int(atlas_summary.get("actionable_owner_gap_count", atlas_summary.get("owner_gap_count", 0)) or 0) actionable_stale = int(atlas_summary.get("actionable_stale_count", atlas_summary.get("stale_count", 0)) or 0) - atlas_issues = actionable_route_collisions + actionable_owner_gaps + actionable_stale + actionable_drift = int(atlas_summary.get("actionable_drift_signal_count", atlas_summary.get("drift_signal_count", 0)) or 0) + atlas_issues = actionable_route_collisions + actionable_owner_gaps + actionable_stale + actionable_drift if not atlas: atlas_status = "warn" atlas_detail = "skill atlas is missing; portfolio-level conflicts are unknown" @@ -339,6 +340,7 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An f"{actionable_route_collisions} actionable route collisions; " f"{actionable_owner_gaps} actionable owner gaps; " f"{actionable_stale} actionable stale; " + f"{actionable_drift} actionable drift; " f"{atlas_summary.get('non_actionable_issue_count', 0)} scoped non-actionable issues" ) gates.append( diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index b5fe17b..05c347f 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -146,7 +146,16 @@ ], "atlas_scope": "release", "actionable": true, - "scope_reason": "default release-actionable skill" + "scope_reason": "default release-actionable skill", + "telemetry": { + "report_present": true, + "report": "reports/adoption_drift_report.json", + "risk_band": "low", + "event_count": 1, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "release-orchestrator", @@ -204,7 +213,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/complex-release-orchestrator/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -243,7 +261,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/final/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -266,7 +293,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v0/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -304,7 +340,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/evolution-frontend-review/v1/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "incident-command-governor", @@ -345,7 +390,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/governed-incident-command/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "note-cleanup", @@ -382,7 +436,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/simple-note-cleanup/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "frontend-review", @@ -422,7 +485,16 @@ ], "atlas_scope": "example", "actionable": false, - "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package.", + "telemetry": { + "report_present": false, + "report": "examples/team-frontend-review/generated-skill/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "geo-ranking-article-generator", @@ -482,7 +554,16 @@ ], "atlas_scope": "embedded-generated-skill", "actionable": false, - "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release.", + "telemetry": { + "report_present": false, + "report": "geo-ranking-article-generator/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "invalid-governance-skill", @@ -504,7 +585,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/governance_invalid_manifest/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-yaml-skill", @@ -528,7 +618,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_invalid_yaml/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } }, { "name": "broken-skill", @@ -551,7 +650,16 @@ ], "atlas_scope": "test-fixture", "actionable": false, - "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers.", + "telemetry": { + "report_present": false, + "report": "tests/fixtures/package_missing_interface_field/reports/adoption_drift_report.json", + "risk_band": "missing", + "event_count": 0, + "adoption_sample_count": 0, + "adoption_rate": 0, + "candidate_count": 0 + } } ], "summary": { @@ -565,6 +673,9 @@ "actionable_stale_count": 0, "shared_resource_count": 0, "no_route_opportunity_count": 0, + "telemetry_report_count": 1, + "drift_signal_count": 0, + "actionable_drift_signal_count": 0, "non_actionable_issue_count": 24 } } diff --git a/skill_atlas/drift_signals.json b/skill_atlas/drift_signals.json new file mode 100644 index 0000000..fe51488 --- /dev/null +++ b/skill_atlas/drift_signals.json @@ -0,0 +1 @@ +[] diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 9a822e4..2f5450e 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -276,6 +276,7 @@ def main() -> None: atlas_gate = next(item for item in payload["gates"] if item["key"] == "skill-atlas") assert atlas_gate["status"] == "pass", atlas_gate assert "actionable route collisions" in atlas_gate["detail"], atlas_gate + assert "actionable drift" in atlas_gate["detail"], atlas_gate operations_gate = next(item for item in payload["gates"] if item["key"] == "operations-loop") assert operations_gate["status"] == "pass", operations_gate assert "metadata events" in operations_gate["detail"], operations_gate @@ -307,6 +308,7 @@ def main() -> None: action_keys = {item["gate_key"] for item in full_payload["review_actions"]} assert action_keys == {"output-lab", "review-waivers"}, full_payload["review_actions"] assert full_payload["data"]["atlas"]["summary"]["actionable_route_collision_count"] == 0, full_payload["data"]["atlas"] + assert full_payload["data"]["atlas"]["summary"]["actionable_drift_signal_count"] == 0, full_payload["data"]["atlas"] assert full_payload["data"]["atlas"]["summary"]["non_actionable_issue_count"] >= 1, full_payload["data"]["atlas"] synthetic_actions = review_studio.build_review_actions( [ diff --git a/tests/verify_skill_atlas.py b/tests/verify_skill_atlas.py index e2381df..8d2e979 100644 --- a/tests/verify_skill_atlas.py +++ b/tests/verify_skill_atlas.py @@ -43,13 +43,54 @@ description: {description} return skill_dir +def write_adoption_report(skill_dir: Path, risk_band: str = "medium") -> None: + reports = skill_dir / "reports" + reports.mkdir(parents=True, exist_ok=True) + (reports / "adoption_drift_report.json").write_text( + json.dumps( + { + "ok": True, + "summary": { + "event_count": 4, + "adoption_sample_count": 3, + "activation_count": 2, + "accepted_count": 1, + "edited_count": 0, + "rejected_count": 1, + "missed_count": 1, + "failed_count": 1, + "adoption_rate": 33.3, + "missed_trigger_count": 1, + "wrong_trigger_count": 0, + "bad_output_count": 1, + "script_error_count": 1, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": risk_band, + }, + "next_iteration_candidates": [ + { + "signal": "bad output", + "recommendation": "Convert the failed output shape into an Output Eval assertion.", + } + ], + "failures": [], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + def main() -> None: tmp_root = ROOT / "tests" / "tmp_skill_atlas" if tmp_root.exists(): shutil.rmtree(tmp_root) tmp_root.mkdir(parents=True, exist_ok=True) - write_skill( + alpha = write_skill( tmp_root, "alpha-release-skill", "Use when turning release notes and changelog snippets into a reusable release brief for teams.", @@ -63,7 +104,7 @@ def main() -> None: }, shared_reference=True, ) - write_skill( + beta = write_skill( tmp_root, "beta-release-skill", "Use when turning changelog notes and release snippets into reusable release briefs for team review.", @@ -75,6 +116,7 @@ def main() -> None: }, shared_reference=True, ) + write_adoption_report(beta, risk_band="high") write_skill( tmp_root, "gamma-doc-skill", @@ -149,19 +191,29 @@ def main() -> None: assert summary["stale_count"] >= 1, summary assert summary["shared_resource_count"] >= 1, summary assert summary["no_route_opportunity_count"] >= 1, summary + assert summary["drift_signal_count"] == 2, summary + assert summary["actionable_drift_signal_count"] == 1, summary + assert summary["telemetry_report_count"] == 1, summary assert (output_dir / "catalog.json").exists(), output_dir assert (output_dir / "route_overlap_matrix.csv").exists(), output_dir assert (output_dir / "dependency_graph.json").exists(), output_dir assert (output_dir / "stale_skills.json").exists(), output_dir assert (output_dir / "owner_review_gaps.json").exists(), output_dir + assert (output_dir / "drift_signals.json").exists(), output_dir assert (output_dir / "no_route_opportunities.json").exists(), output_dir html = report_html.read_text(encoding="utf-8") assert "Skill Atlas" in html, html[:1000] assert "Route Collisions" in html, html[:3000] assert "Actionable Issues" in html, html[:3000] + assert "Drift Signals" in html, html[:3000] assert "No-Route Opportunities" in html, html[:3000] catalog = json.loads((output_dir / "catalog.json").read_text(encoding="utf-8")) assert catalog["summary"]["skill_count"] == 3, catalog + alpha_catalog = next(item for item in catalog["skills"] if item["name"] == alpha.name) + assert alpha_catalog["telemetry"]["risk_band"] == "missing", alpha_catalog + drift = json.loads((output_dir / "drift_signals.json").read_text(encoding="utf-8")) + assert any(item["name"] == alpha.name and item["signal_types"] == ["no telemetry"] for item in drift), drift + assert any(item["name"] == beta.name and "bad output" in item["signal_types"] and not item["actionable"] for item in drift), drift assert payload["scope_policy"]["present"], payload["scope_policy"] print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))