diff --git a/AGENTS.md b/AGENTS.md index 9b9b709c..8ca76355 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -71,6 +71,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/yao_cli_parser.py`: CLI argparse command surface, flags, choices, and command handler binding. - `scripts/yao_cli_telemetry.py`: opt-in metadata-only CLI run telemetry. Keep it free of prompt, argument, output, transcript, note, or message capture. - `scripts/import_telemetry_events.py`: external telemetry importer. Validate the whole input before appending events, and keep raw prompt/output/transcript/message/note fields blocked. +- `scripts/emit_telemetry_event.py`: external client telemetry emitter. It may append one normalized metadata event to a local spool, but must never accept or write raw prompt, output, transcript, message, note, argument, or private content. New helper modules that are imported by CLI/report scripts but are not standalone commands must declare: diff --git a/Makefile b/Makefile index 458d1c22..12a78243 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check telemetry-import-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -94,6 +94,9 @@ adoption-drift-check: telemetry-import-check: $(PYTHON) tests/verify_telemetry_import.py +telemetry-emit-check: + $(PYTHON) tests/verify_telemetry_emit.py + review-waivers-check: $(PYTHON) tests/verify_review_waivers.py @@ -190,11 +193,11 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check telemetry-import-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 5bdde0aa..18a06451 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a systems-thinking model that maps boundaries, feedback loops, drift risks, recurring failure patterns, and highest-leverage quality moves - three high-value next iteration directions after the first package is created - a lightweight feedback log that does not require a full promotion cycle -- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates, with optional `yao.py` CLI run capture and external client JSONL import that record command names and outcomes without arguments or raw content +- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates, with optional `yao.py` CLI run capture, external client event emit hooks, and JSONL import that record command names and outcomes without arguments or raw content - a Skill Atlas drift layer that reads aggregate adoption reports and surfaces portfolio-level drift signals without packaging raw telemetry logs - a baseline compare report for with-skill vs baseline review - a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits @@ -109,6 +109,7 @@ python3 scripts/yao.py system-model my-skill python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before adding scripts." --rating 4 --category boundary python3 scripts/yao.py adoption-drift my-skill --record-event skill_activation --activation-type explicit --outcome accepted YAO_CLI_TELEMETRY=1 python3 scripts/yao.py validate my-skill +python3 scripts/yao.py telemetry-emit my-skill --event skill_activation --activation-type explicit --outcome accepted --command browser-extension python3 scripts/yao.py telemetry-import my-skill --input-jsonl /tmp/external-client-events.jsonl --command browser-extension python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key trust-report --reviewer "Yao Team" --reason "Known warning accepted for this release with bounded follow-up." --expires-at 2026-09-30 python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key permission-gates --reviewer "Yao Team" --reason "Permission warning accepted only for this non-governed release window." --expires-at 2026-09-30 @@ -388,7 +389,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, telemetry-emit, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -416,6 +417,7 @@ Utility scripts that make the meta-skill operational: - `upgrade_check.py`: compares current and previous registry package metadata, recommends a version bump, and blocks incompatible upgrade claims - `render_adoption_drift_report.py`: records metadata-only local telemetry and renders adoption, missed-trigger, bad-output, script-error, and review-drift signals without packaging raw event logs - `import_telemetry_events.py`: imports external metadata-only telemetry JSONL after whole-file privacy validation, then refreshes the aggregate adoption drift report +- `emit_telemetry_event.py`: emits one metadata-only external client event into a local spool for later `telemetry-import`, with dry-run validation and raw-content field blocking - `yao_cli_telemetry.py`: opt-in metadata-only `yao.py` run capture for command name, source, outcome, and failure class without command arguments or raw content - `render_review_waivers.py`: validates human reviewer risk approvals with gate keys, reasons, expiry dates, and blocker-safe waiver policy - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain diff --git a/references/telemetry-drift-method.md b/references/telemetry-drift-method.md index 6e443c03..2ef0dbb5 100644 --- a/references/telemetry-drift-method.md +++ b/references/telemetry-drift-method.md @@ -61,6 +61,37 @@ python3 scripts/yao.py --no-cli-telemetry validate . Successful CLI runs record `event=script_run`, `source=yao_cli`, `outcome=accepted`, and `failure_type=none`. Failed CLI runs record `outcome=failed` and `failure_type=script_error`. The command name is normalized to the subcommand only; command arguments are never recorded. +## External Client Emit + +External clients, browser extensions, editor adapters, or wrapper scripts can emit one sanitized event at a time into a local spool before importing it into the aggregate drift report: + +```bash +python3 scripts/yao.py telemetry-emit . \ + --event skill_activation \ + --activation-type explicit \ + --outcome accepted \ + --command browser-extension +``` + +By default this writes to `.yao/telemetry_spool/external_events.jsonl`. Use `--output-jsonl` when a client needs a different local handoff path: + +```bash +python3 scripts/yao.py telemetry-emit . \ + --output-jsonl /tmp/external-client-events.jsonl \ + --event skill_output \ + --activation-type manual \ + --outcome edited \ + --command browser-plugin +``` + +Use `--dry-run` to validate a proposed event without writing to the spool. The emitter uses the same metadata-only contract as import: no prompt, input, output, transcript, message, note, raw text, arguments, or unknown fields are accepted. + +After a client finishes a batch, import the spool: + +```bash +python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl +``` + ## External Client Import External clients, browser extensions, editor adapters, or wrapper scripts may hand off already-sanitized JSONL through `telemetry-import`: @@ -97,7 +128,7 @@ Package builders should exclude `reports/telemetry_events.jsonl`. The root repos ## Iteration Loop -1. Capture metadata-only events locally, either manually with `adoption-drift --record-event`, automatically with opt-in `yao.py` CLI capture, or through validated external JSONL import. +1. Capture metadata-only events locally, either manually with `adoption-drift --record-event`, automatically with opt-in `yao.py` CLI capture, through `telemetry-emit` client hooks, or through validated external JSONL import. 2. Render `reports/adoption_drift_report.md`. 3. Convert missed triggers into trigger eval cases. 4. Convert bad outputs into Output Eval assertions and failure taxonomy entries. diff --git a/registry/index.json b/registry/index.json index 5807dffe..474b7496 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 6efeb49f..eaba237e 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9", - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 49f99882..240fac7f 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T13:45:00Z", + "generated_at": "2026-06-13T14:00:00Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index 11aa4435..d81a1ef0 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -156,6 +156,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -266,7 +267,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -372,7 +373,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -383,6 +384,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -458,14 +460,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -487,7 +489,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -508,7 +510,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -519,7 +521,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -600,7 +602,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -611,6 +613,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -686,14 +689,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -713,7 +716,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -734,7 +737,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -745,7 +748,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -958,6 +961,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -1068,7 +1072,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -1174,7 +1178,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1185,6 +1189,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -1260,14 +1265,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1289,7 +1294,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -1310,7 +1315,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -1321,7 +1326,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -1402,7 +1407,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1413,6 +1418,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -1488,14 +1494,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1515,7 +1521,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -1536,7 +1542,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -1547,7 +1553,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -1760,6 +1766,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -1870,7 +1877,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -1976,7 +1983,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1987,6 +1994,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -2062,14 +2070,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2091,7 +2099,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -2112,7 +2120,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -2122,7 +2130,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -2197,7 +2205,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2208,6 +2216,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -2283,14 +2292,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2310,7 +2319,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -2331,7 +2340,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -2341,7 +2350,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -2546,6 +2555,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -2656,7 +2666,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -2762,7 +2772,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2773,6 +2783,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -2848,14 +2859,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2877,7 +2888,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -2898,7 +2909,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -2909,7 +2920,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -2983,7 +2994,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2994,6 +3005,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -3069,14 +3081,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3096,7 +3108,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -3117,7 +3129,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -3128,7 +3140,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -3332,6 +3344,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -3442,7 +3455,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -3548,7 +3561,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3559,6 +3572,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -3634,14 +3648,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3663,7 +3677,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -3684,7 +3698,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -3695,7 +3709,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -3773,7 +3787,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3784,6 +3798,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -3859,14 +3874,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3886,7 +3901,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -3907,7 +3922,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -3918,7 +3933,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { diff --git a/reports/context_budget.json b/reports/context_budget.json index 1e3f4cc5..6026695f 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 922840, + "other_text_tokens": 927304, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 923591, - "relevant_file_count": 398, + "estimated_total_text_tokens": 928055, + "relevant_file_count": 403, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index e3685deb..0a4121d0 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 497, + "archive_entry_count": 499, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 235b5e3f..e31adb7e 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.38, + "duration_ms": 31.02, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.75, + "duration_ms": 32.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.8, + "duration_ms": 30.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.43, + "duration_ms": 31.2, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.92, + "duration_ms": 28.99, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.23, + "duration_ms": 29.64, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.19, + "duration_ms": 30.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 34.13, + "duration_ms": 28.91, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.55, + "duration_ms": 28.16, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.09, + "duration_ms": 29.25, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index fb489a53..eb4fdfba 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 31.38 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 30.75 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 30.8 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 32.43 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 31.92 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 32.23 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 31.19 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 34.13 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 32.55 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 32.09 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 31.02 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 32.55 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 30.25 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.2 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 28.99 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 29.64 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 30.23 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.91 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.16 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 29.25 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 34a4ba96..5cb7420d 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", - "archive_entry_count": 497, + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", + "archive_entry_count": 499, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 9e7c4235..07fd53d3 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c` +- Archive SHA256: `3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 185a8a67..88037447 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9", - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 8658947d..116a1f62 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9` -- Archive SHA256: `7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c` +- Package SHA256: `195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4` +- Archive SHA256: `3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 549ba4fc..7e3218d8 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -242,12 +242,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

71 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

497 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

72 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

499 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 71 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 72 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -313,7 +313,7 @@
-

信任报告

Secret
0
脚本数
71
网络脚本
3
Help 失败
0
包体哈希
57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9
+

信任报告

Secret
0
脚本数
72
网络脚本
3
Help 失败
0
包体哈希
195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -344,12 +344,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
497
失败数
0
警告数
0
归档哈希
7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
499
失败数
0
警告数
0
归档哈希
3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599
diff --git a/reports/review-studio.json b/reports/review-studio.json index c69c2d3e..34a59953 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 71 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 72 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -355,7 +355,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 350 个词/字，references 约 14978 个词/字。", + "入口约 350 个词/字，references 约 15124 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -459,7 +459,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 350 个词/字，references 约 14978 个词/字。" + "上下文成本需要补强：入口约 350 个词/字，references 约 15124 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -692,7 +692,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 71 + "file_count": 72 }, { "path": "evals", @@ -707,7 +707,7 @@ "file_count": 171 } ], - "file_count": 306, + "file_count": 307, "folder_count": 4, "distribution": [ { @@ -732,7 +732,7 @@ }, { "label": "scripts", - "value": 71 + "value": 72 }, { "label": "evals", @@ -861,7 +861,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 71 + "file_count": 72 }, { "path": "evals", @@ -1207,8 +1207,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 155, - "script_count": 71, + "scanned_files": 156, + "script_count": 72, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -1217,18 +1217,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 48, + "file_write_script_count": 49, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 61, + "help_smoke_checked_count": 62, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 155, - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "package_hash_file_count": 156, + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" }, "skill_atlas": { "skill_count": 12, @@ -1266,8 +1266,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9", - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, "compatibility": { "openai": "pass", @@ -1298,7 +1298,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1314,8 +1314,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", - "archive_entry_count": 497, + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", + "archive_entry_count": 499, "failure_count": 0, "warning_count": 0 }, @@ -1326,7 +1326,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 497, + "archive_entry_count": 499, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1393,12 +1393,12 @@ { "field": "archive_sha256", "from": "", - "to": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "to": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "to": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" } ] }, @@ -3132,7 +3132,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.38, + "duration_ms": 31.02, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3160,7 +3160,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.75, + "duration_ms": 32.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3183,7 +3183,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.8, + "duration_ms": 30.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3211,7 +3211,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.43, + "duration_ms": 31.2, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3234,7 +3234,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.92, + "duration_ms": 28.99, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3262,7 +3262,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.23, + "duration_ms": 29.64, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3285,7 +3285,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.19, + "duration_ms": 30.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3312,7 +3312,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 34.13, + "duration_ms": 28.91, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3335,7 +3335,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.55, + "duration_ms": 28.16, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3364,7 +3364,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.09, + "duration_ms": 29.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3829,6 +3829,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -3939,7 +3940,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -4045,7 +4046,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4056,6 +4057,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -4131,14 +4133,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4160,7 +4162,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -4181,7 +4183,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -4192,7 +4194,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -4273,7 +4275,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4284,6 +4286,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -4359,14 +4362,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4386,7 +4389,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -4407,7 +4410,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -4418,7 +4421,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -4631,6 +4634,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -4741,7 +4745,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -4847,7 +4851,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4858,6 +4862,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -4933,14 +4938,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4962,7 +4967,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -4983,7 +4988,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -4994,7 +4999,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -5075,7 +5080,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5086,6 +5091,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -5161,14 +5167,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5188,7 +5194,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -5209,7 +5215,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -5220,7 +5226,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -5433,6 +5439,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -5543,7 +5550,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -5649,7 +5656,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5660,6 +5667,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -5735,14 +5743,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5764,7 +5772,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -5785,7 +5793,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -5795,7 +5803,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -5870,7 +5878,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5881,6 +5889,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -5956,14 +5965,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5983,7 +5992,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -6004,7 +6013,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -6014,7 +6023,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -6219,6 +6228,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -6329,7 +6339,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -6435,7 +6445,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6446,6 +6456,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -6521,14 +6532,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6550,7 +6561,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -6571,7 +6582,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -6582,7 +6593,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -6656,7 +6667,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6667,6 +6678,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -6742,14 +6754,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6769,7 +6781,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -6790,7 +6802,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -6801,7 +6813,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -7005,6 +7017,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -7115,7 +7128,7 @@ ], "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 } @@ -7221,7 +7234,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7232,6 +7245,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -7307,14 +7321,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7336,7 +7350,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -7357,7 +7371,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -7368,7 +7382,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -7446,7 +7460,7 @@ }, "file_write": { "required": true, - "script_count": 48, + "script_count": 49, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7457,6 +7471,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -7532,14 +7547,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 61, + "checked_count": 62, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 48, + "file_write_script_count": 49, "subprocess_script_count": 7, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7559,7 +7574,7 @@ ], "capability_counts": { "network": 3, - "file_write": 48, + "file_write": 49, "subprocess": 7, "interactive": 0 }, @@ -7580,7 +7595,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 71, + "scripts": 72, "assets": 2, "reports": 41 }, @@ -7591,7 +7606,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 71, + "script_count": 72, "help_smoke_failed_count": 0 }, "permissions": { @@ -8231,8 +8246,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 155, - "script_count": 71, + "scanned_files": 156, + "script_count": 72, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -8241,18 +8256,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 48, + "file_write_script_count": 49, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 61, + "help_smoke_checked_count": 62, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 155, - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "package_hash_file_count": 156, + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" }, "failures": [], "warnings": [], @@ -8417,6 +8432,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/emit_telemetry_event.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/export_skill_ir.py", "interface": "cli", @@ -9300,9 +9329,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 61, - "checked_count": 61, - "passed_count": 61, + "candidate_count": 62, + "checked_count": 62, + "passed_count": 62, "failed_count": 0, "skipped_count": 10, "failed_scripts": [], @@ -9417,6 +9446,16 @@ "stdout_excerpt": "usage: diff_eval.py [-h] before after\n\nDiff two eval reports.\n\npositional arguments:\n before\n after\n\noptions:\n -h, --help show this help message and exit", "stderr_excerpt": "" }, + { + "path": "scripts/emit_telemetry_event.py", + "command": "python3 scripts/emit_telemetry_event.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: emit_telemetry_event.py [-h] [--output-jsonl OUTPUT_JSONL]\n [--event {review_event,script_run,skill_activation,skill_output}]\n [--activation-type {explicit,implicit,manual,u", + "stderr_excerpt": "" + }, { "path": "scripts/export_skill_ir.py", "command": "python3 scripts/export_skill_ir.py --help", @@ -10022,6 +10061,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", @@ -10127,10 +10167,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 922840, + "other_text_tokens": 927304, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 923591, - "relevant_file_count": 398, + "estimated_total_text_tokens": 928055, + "relevant_file_count": 403, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -11005,6 +11045,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", @@ -11960,7 +12001,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T13:45:00Z", + "generated_at": "2026-06-13T14:00:00Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -12128,8 +12169,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9", - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, "compatibility": { "openai": "pass", @@ -12160,7 +12201,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -12185,7 +12226,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" } ] }, @@ -12208,8 +12249,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", - "archive_entry_count": 497, + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", + "archive_entry_count": 499, "failure_count": 0, "warning_count": 0 }, @@ -12874,7 +12915,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 497, + "archive_entry_count": 499, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -13216,12 +13257,12 @@ { "field": "archive_sha256", "from": "", - "to": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "to": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "to": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index e719ebcc..beb86c66 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 155, - "script_count": 71, + "scanned_files": 156, + "script_count": 72, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 48, + "file_write_script_count": 49, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 61, + "help_smoke_checked_count": 62, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 155, - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "package_hash_file_count": 156, + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" }, "failures": [], "warnings": [], @@ -188,6 +188,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/emit_telemetry_event.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/export_skill_ir.py", "interface": "cli", @@ -1071,9 +1085,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 61, - "checked_count": 61, - "passed_count": 61, + "candidate_count": 62, + "checked_count": 62, + "passed_count": 62, "failed_count": 0, "skipped_count": 10, "failed_scripts": [], @@ -1188,6 +1202,16 @@ "stdout_excerpt": "usage: diff_eval.py [-h] before after\n\nDiff two eval reports.\n\npositional arguments:\n before\n after\n\noptions:\n -h, --help show this help message and exit", "stderr_excerpt": "" }, + { + "path": "scripts/emit_telemetry_event.py", + "command": "python3 scripts/emit_telemetry_event.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: emit_telemetry_event.py [-h] [--output-jsonl OUTPUT_JSONL]\n [--event {review_event,script_run,skill_activation,skill_output}]\n [--activation-type {explicit,implicit,manual,u", + "stderr_excerpt": "" + }, { "path": "scripts/export_skill_ir.py", "command": "python3 scripts/export_skill_ir.py --help", @@ -1793,6 +1817,7 @@ "scripts/compile_skill.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/import_telemetry_events.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 966df4a7..3243b770 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `155` -- Scripts: `71` +- Scanned files: `156` +- Scripts: `72` - Internal script modules: `10` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `48` +- File-write scripts: `49` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `61` +- CLI help smoke checked: `62` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `155` -- Package SHA256: `57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9` +- Package hash files: `156` +- Package SHA256: `195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `61` -- Passed scripts: `61` +- Checked scripts: `62` +- Passed scripts: `62` - Failed scripts: `none` ## Script Surface @@ -73,6 +73,7 @@ | scripts/create_iteration_snapshot.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/cross_packager.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/diff_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/emit_telemetry_event.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/export_skill_ir.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/github_benchmark_scan.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/governance_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index c72fb3ac..b39ea19b 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -24,6 +24,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, iteration candidate generation, and Skill Atlas drift-signal input. - CLI Telemetry Capture v0 for opt-in `yao.py` command-run metadata that records command name, source, outcome, failure class, and timestamp without command arguments or raw user/model content. - External Telemetry Import v0 for whole-file validated metadata-only JSONL ingestion from non-CLI clients, with dry-run support and all-or-nothing rejection on raw content fields or schema violations. +- Telemetry Emit Hook v0 for Browser/Chrome/IDE/wrapper integrations to append one validated metadata-only external event into a local spool before import, with dry-run validation and raw-content blocking. - Review Waivers v0 for human warning acceptance with reviewer, reason, scope, expiry, and blocker-safe policy. - Governed Permission Gates v0 for reviewer-approved network, file-write, and subprocess capabilities with scope, reason, expiry, evidence, and target-enforcement mapping. - Runtime Permission Probes v0 for packaged target adapter checks, explicit native-enforcement flags, metadata fallback evidence, and residual permission risks. @@ -35,7 +36,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case. - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. -This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, validated external JSONL imports can bring non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, native client telemetry hooks, provider-native installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, external clients can now emit one validated metadata-only event into a local spool, validated external JSONL imports can bring those non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, platform-native automatic client telemetry integrations, provider-native installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open. ## Coverage Matrix @@ -56,7 +57,7 @@ This is still not the final world-class state. Target-native behavior contracts | Install Simulation | `scripts/simulate_install.py`, `reports/install_simulation.md`, `tests/verify_install_simulation.py` with installer permission coverage checks | v0 landed | | Local Install Sync Preflight | `scripts/sync_local_install.py`, `tests/verify_local_install_sync.py`, `Makefile` package-check prerequisites | v0 landed | | Upgrade Check | `scripts/upgrade_check.py`, `reports/upgrade_check.md`, `tests/verify_upgrade_check.py` | v0 landed | -| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `scripts/yao_cli_telemetry.py`, `scripts/import_telemetry_events.py`, `reports/adoption_drift_report.md`, `tests/verify_adoption_drift.py`, `tests/verify_telemetry_import.py`, `tests/verify_yao_cli.py`, `references/telemetry-drift-method.md` | v0 landed | +| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `scripts/yao_cli_telemetry.py`, `scripts/emit_telemetry_event.py`, `scripts/import_telemetry_events.py`, `reports/adoption_drift_report.md`, `tests/verify_adoption_drift.py`, `tests/verify_telemetry_emit.py`, `tests/verify_telemetry_import.py`, `tests/verify_yao_cli.py`, `references/telemetry-drift-method.md` | v0 landed | | Review Waivers | `scripts/render_review_waivers.py`, `reports/review_waivers.md`, `tests/verify_review_waivers.py`, `references/review-waiver-method.md` | v0 landed | | Governed Permission Gates | `security/permission_policy.json`, `scripts/trust_check.py`, `scripts/render_review_studio.py`, `tests/verify_trust_check.py`, `tests/verify_review_studio.py` | v0 landed | | Runtime Permission Probes | `scripts/probe_runtime_permissions.py`, `reports/runtime_permission_probes.md`, `tests/verify_runtime_permission_probes.py`, `tests/verify_review_studio.py` | v0 landed | @@ -84,11 +85,11 @@ The Review Studio page aggregates intent, trigger, output, context, runtime, tru Next move: add richer source-line anchors inside generated reports and record real reviewer annotations during the next human review pass. -### 4. Multi-skill operation now links Atlas with drift, but needs real client capture +### 4. Multi-skill operation now links Atlas with drift, but needs platform-native capture -The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, aggregate drift signals, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, feed next iteration candidates into Review Studio, and publish aggregate drift input for Atlas. `yao.py` now adds opt-in automatic CLI run capture through `YAO_CLI_TELEMETRY=1` or `--record-cli-telemetry`, recording only `source=yao_cli`, normalized subcommand name, outcome, failure class, and timestamp. `telemetry-import` now accepts already-sanitized external client JSONL, validates the whole file before appending, supports dry-run, defaults `source=external`, and rejects any raw content or unknown metadata fields without mutating the local event stream. It still needs native client hooks that produce those external JSONL batches automatically. +The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, aggregate drift signals, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, feed next iteration candidates into Review Studio, and publish aggregate drift input for Atlas. `yao.py` now adds opt-in automatic CLI run capture through `YAO_CLI_TELEMETRY=1` or `--record-cli-telemetry`, recording only `source=yao_cli`, normalized subcommand name, outcome, failure class, and timestamp. `telemetry-emit` gives Browser/Chrome/IDE/wrapper integrations a stable local hook for appending one normalized external event into `.yao/telemetry_spool/external_events.jsonl`, with dry-run validation and no raw content fields. `telemetry-import` accepts already-sanitized external client JSONL or the emitted spool, validates the whole file before appending, supports dry-run, defaults `source=external`, and rejects any raw content or unknown metadata fields without mutating the local event stream. It still needs platform-native integrations that call the emit hook automatically from real clients. -Next move: wire native client hooks to emit the external metadata-only JSONL batches automatically, then let Atlas rank stale, drifting, or conflicting skills by real usage impact. +Next move: wire Browser, Chrome, IDE, and provider-native integrations to call the emit hook automatically, then let Atlas rank stale, drifting, or conflicting skills by real usage impact. ### 5. Trust report is structural, not full security review @@ -103,22 +104,22 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `61 / 61` CLI help smoke checks passing across `71` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `62 / 62` CLI help smoke checks passing across `72` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `497` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `497` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `499` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `499` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | -| Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | +| Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | | Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0 | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | | Review Studio | decision `review`, world-class score `92`, `13` gates, `0` blockers, `2` warnings, `2` review actions, `0` open annotation blockers | | IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable | | Context Budget | initial load `944/1000`, under the production budget | -| CI | `make ci-test` target count is `57` after the telemetry import gate update | +| CI | `make ci-test` target count is `58` after the telemetry emit hook gate update | ## Next Highest-Leverage Moves @@ -126,4 +127,4 @@ Next move: add real client or installer permission enforcement integration. 2. Add native client or installer runtime enforcement for approved high-permission capabilities. 3. Run the new provider-backed output runner against holdout cases with real credentials, then record the current blind A/B decisions before claiming fully ready status. 4. Add real reviewer annotation records during the next human review pass. -5. Wire native client telemetry hooks so external JSONL batches are produced automatically, not only imported from already-sanitized files. +5. Wire platform-native clients to call `telemetry-emit` automatically, not only through wrapper scripts. diff --git a/reports/skill-overview.html b/reports/skill-overview.html index dab43581..8a14c933 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -609,7 +609,7 @@
-
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 350 个词/字，references 约 14978 个词/字。入口约 350 个词/字，references 约 14978 个词/字。
+
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 350 个词/字，references 约 15124 个词/字。入口约 350 个词/字，references 约 15124 个词/字。
@@ -641,10 +641,10 @@

指标判读Reading

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 350 个词/字，references 约 14978 个词/字。入口约 350 个词/字，references 约 14978 个词/字。
+
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 350 个词/字，references 约 15124 个词/字。入口约 350 个词/字，references 约 15124 个词/字。
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 350 个词/字，references 约 14978 个词/字。入口约 350 个词/字，references 约 14978 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 350 个词/字，references 约 15124 个词/字。入口约 350 个词/字，references 约 15124 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
@@ -720,7 +720,7 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 350 个词/字，references 约 14978 个词/字。上下文成本需要补强：入口约 350 个词/字，references 约 14978 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 350 个词/字，references 约 15124 个词/字。上下文成本需要补强：入口约 350 个词/字，references 约 15124 个词/字。纳入下一轮修复Fix next
@@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布306项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布307项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 90c01027..03e940e4 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -96,7 +96,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 350 个词/字，references 约 14978 个词/字。", + "入口约 350 个词/字，references 约 15124 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -200,7 +200,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 350 个词/字，references 约 14978 个词/字。" + "上下文成本需要补强：入口约 350 个词/字，references 约 15124 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -433,7 +433,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 71 + "file_count": 72 }, { "path": "evals", @@ -448,7 +448,7 @@ "file_count": 171 } ], - "file_count": 306, + "file_count": 307, "folder_count": 4, "distribution": [ { @@ -473,7 +473,7 @@ }, { "label": "scripts", - "value": 71 + "value": 72 }, { "label": "evals", @@ -602,7 +602,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 71 + "file_count": 72 }, { "path": "evals", @@ -948,8 +948,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 155, - "script_count": 71, + "scanned_files": 156, + "script_count": 72, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -958,18 +958,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 48, + "file_write_script_count": 49, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 61, + "help_smoke_checked_count": 62, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 155, - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "package_hash_file_count": 156, + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" }, "skill_atlas": { "skill_count": 12, @@ -1007,8 +1007,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9", - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "package_sha256": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, "compatibility": { "openai": "pass", @@ -1039,7 +1039,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1055,8 +1055,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c", - "archive_entry_count": 497, + "archive_sha256": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599", + "archive_entry_count": 499, "failure_count": 0, "warning_count": 0 }, @@ -1067,7 +1067,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 497, + "archive_entry_count": 499, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1134,12 +1134,12 @@ { "field": "archive_sha256", "from": "", - "to": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "to": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "to": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" } ] }, diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index c14d0a8b..a2eb4bc5 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -75,6 +75,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index a1fe1819..fd4e7e8a 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c" + "to": "3c6223c609ef538fc3ff9a100c3896894cd8d443a304dcd77524d88a22e02599" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9" + "to": "195c77ccf18eb0dd72c429b4cbe6137de531a34c00a4ce4a97f9385b3a3b26f4" } ] }, diff --git a/scripts/ci_test.py b/scripts/ci_test.py index c3e179af..55d7eb7e 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -51,6 +51,7 @@ DEFAULT_TARGETS = [ "feedback-check", "adoption-drift-check", "telemetry-import-check", + "telemetry-emit-check", "review-waivers-check", "review-annotations-check", "baseline-compare-check", diff --git a/scripts/emit_telemetry_event.py b/scripts/emit_telemetry_event.py new file mode 100644 index 00000000..d737703c --- /dev/null +++ b/scripts/emit_telemetry_event.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +import argparse +import json +import shlex +from pathlib import Path +from typing import Any + +from render_adoption_drift_report import ( + ALLOWED_ACTIVATION_TYPES, + ALLOWED_EVENTS, + ALLOWED_FAILURE_TYPES, + ALLOWED_OUTCOMES, + ALLOWED_SOURCES, + display_path, + normalize_event, + skill_defaults, +) + + +def default_spool_path(skill_dir: Path) -> Path: + return skill_dir / ".yao" / "telemetry_spool" / "external_events.jsonl" + + +def append_event(path: Path, event: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n") + + +def import_command(skill_dir: Path, output_jsonl: Path) -> str: + return shlex.join( + [ + "python3", + "scripts/yao.py", + "telemetry-import", + display_path(skill_dir), + "--input-jsonl", + display_path(output_jsonl), + ] + ) + + +def emit_event( + skill_dir: Path, + output_jsonl: Path | None = None, + event_name: str = "script_run", + activation_type: str = "manual", + outcome: str = "unknown", + failure_type: str = "none", + source: str = "external", + command: str = "external-client", + timestamp: str | None = None, + skill_name: str | None = None, + version: str | None = None, + dry_run: bool = False, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + output_jsonl = (output_jsonl or default_spool_path(skill_dir)).resolve() + raw_event: dict[str, Any] = { + "event": event_name, + "activation_type": activation_type, + "outcome": outcome, + "failure_type": failure_type, + "source": source, + "command": command, + } + if timestamp: + raw_event["timestamp"] = timestamp + if skill_name: + raw_event["skill"] = skill_name + if version: + raw_event["version"] = version + event, failures = normalize_event(raw_event, skill_defaults(skill_dir), "emit") + if event and not failures and not dry_run: + append_event(output_jsonl, event) + return { + "ok": not failures, + "schema_version": "1.0", + "skill_dir": display_path(skill_dir), + "output_jsonl": display_path(output_jsonl), + "dry_run": dry_run, + "emitted": bool(event and not failures and not dry_run), + "event": event or {}, + "failures": failures, + "artifacts": { + "spool_jsonl": display_path(output_jsonl), + "import_command": import_command(skill_dir, output_jsonl), + }, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Emit one metadata-only telemetry event for later import.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-jsonl") + parser.add_argument("--event", choices=sorted(ALLOWED_EVENTS), default="script_run") + parser.add_argument("--activation-type", choices=sorted(ALLOWED_ACTIVATION_TYPES), default="manual") + parser.add_argument("--outcome", choices=sorted(ALLOWED_OUTCOMES), default="unknown") + parser.add_argument("--failure-type", choices=sorted(ALLOWED_FAILURE_TYPES), default="none") + parser.add_argument("--source", choices=sorted(ALLOWED_SOURCES), default="external") + parser.add_argument("--command", default="external-client") + parser.add_argument("--timestamp") + parser.add_argument("--skill-name") + parser.add_argument("--version") + parser.add_argument("--dry-run", action="store_true") + args = parser.parse_args() + + report = emit_event( + Path(args.skill_dir), + output_jsonl=Path(args.output_jsonl) if args.output_jsonl else None, + event_name=args.event, + activation_type=args.activation_type, + outcome=args.outcome, + failure_type=args.failure_type, + source=args.source, + command=args.command, + timestamp=args.timestamp, + skill_name=args.skill_name, + version=args.version, + dry_run=args.dry_run, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/yao.py b/scripts/yao.py index 8227a8ea..0b7389c5 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -727,6 +727,38 @@ def command_telemetry_import(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_telemetry_emit(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [ + skill_dir, + "--event", + args.event, + "--activation-type", + args.activation_type, + "--outcome", + args.outcome, + "--failure-type", + args.failure_type, + "--source", + args.source, + "--command", + args.telemetry_command, + ] + if args.output_jsonl: + cmd.extend(["--output-jsonl", args.output_jsonl]) + if args.timestamp: + cmd.extend(["--timestamp", args.timestamp]) + if args.skill_name: + cmd.extend(["--skill-name", args.skill_name]) + if args.version: + cmd.extend(["--version", args.version]) + if args.dry_run: + cmd.append("--dry-run") + result = run_script("emit_telemetry_event.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_review_waivers(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index eb28884e..765f6eec 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -299,6 +299,48 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] telemetry_import_cmd.add_argument("--dry-run", action="store_true") telemetry_import_cmd.set_defaults(func=_handler(command_handlers, "command_telemetry_import")) + telemetry_emit_cmd = subparsers.add_parser( + "telemetry-emit", + help="Emit one metadata-only telemetry event for later import.", + ) + telemetry_emit_cmd.add_argument("skill_dir", nargs="?", default=".") + telemetry_emit_cmd.add_argument("--output-jsonl") + telemetry_emit_cmd.add_argument( + "--event", + choices=["review_event", "script_run", "skill_activation", "skill_output"], + default="script_run", + ) + telemetry_emit_cmd.add_argument( + "--activation-type", + choices=["explicit", "implicit", "manual", "unknown"], + default="manual", + ) + telemetry_emit_cmd.add_argument( + "--outcome", + choices=["accepted", "edited", "failed", "missed", "rejected", "reviewed", "unknown"], + default="unknown", + ) + telemetry_emit_cmd.add_argument( + "--failure-type", + choices=[ + "bad_output", + "missing_resource", + "none", + "review_overdue", + "script_error", + "under_trigger", + "wrong_trigger", + ], + default="none", + ) + telemetry_emit_cmd.add_argument("--source", choices=["external", "manual", "unknown", "yao_cli"], default="external") + telemetry_emit_cmd.add_argument("--command", dest="telemetry_command", default="external-client") + telemetry_emit_cmd.add_argument("--timestamp") + telemetry_emit_cmd.add_argument("--skill-name") + telemetry_emit_cmd.add_argument("--version") + telemetry_emit_cmd.add_argument("--dry-run", action="store_true") + telemetry_emit_cmd.set_defaults(func=_handler(command_handlers, "command_telemetry_emit")) + review_waivers_cmd = subparsers.add_parser( "review-waivers", help="Render or update human reviewer waiver evidence for Review Studio.", diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index ea787eb6..c4fb2cca 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -119,6 +119,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index bc33d3d4..fb72f160 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -31,6 +31,7 @@ "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", "scripts/diff_eval.py", + "scripts/emit_telemetry_event.py", "scripts/export_skill_ir.py", "scripts/github_benchmark_scan.py", "scripts/governance_check.py", diff --git a/tests/verify_telemetry_emit.py b/tests/verify_telemetry_emit.py new file mode 100644 index 00000000..bb03789a --- /dev/null +++ b/tests/verify_telemetry_emit.py @@ -0,0 +1,172 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_telemetry_emit" +EMITTER = ROOT / "scripts" / "emit_telemetry_event.py" +YAO = ROOT / "scripts" / "yao.py" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + + +def read_jsonl(path: Path) -> list[dict]: + if not path.exists(): + return [] + return [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()] + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + TMP.mkdir(parents=True, exist_ok=True) + + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "telemetry-emit-demo", + "--description", + "Emit metadata-only telemetry from an external client hook for later adoption drift import.", + "--output-dir", + str(TMP), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + skill_dir = TMP / "telemetry-emit-demo" + spool = TMP / "external-spool.jsonl" + + emitted = run( + [ + sys.executable, + str(EMITTER), + str(skill_dir), + "--output-jsonl", + str(spool), + "--event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--failure-type", + "none", + "--command", + "browser-extension", + "--timestamp", + "2026-06-13T12:00:00Z", + ] + ) + assert emitted["ok"], emitted + assert emitted["payload"]["emitted"] is True, emitted + assert emitted["payload"]["artifacts"]["spool_jsonl"].endswith("external-spool.jsonl"), emitted + assert "telemetry-import" in emitted["payload"]["artifacts"]["import_command"], emitted + events = read_jsonl(spool) + assert len(events) == 1, events + assert events[0]["event"] == "skill_activation", events + assert events[0]["source"] == "external", events + assert events[0]["command"] == "browser-extension", events + + dry_run = run( + [ + sys.executable, + str(EMITTER), + str(skill_dir), + "--output-jsonl", + str(spool), + "--event", + "script_run", + "--outcome", + "failed", + "--failure-type", + "script_error", + "--command", + "browser-extension", + "--dry-run", + ] + ) + assert dry_run["ok"], dry_run + assert dry_run["payload"]["dry_run"] is True, dry_run + assert dry_run["payload"]["emitted"] is False, dry_run + assert len(read_jsonl(spool)) == 1, read_jsonl(spool) + + invalid = run( + [ + sys.executable, + str(EMITTER), + str(skill_dir), + "--output-jsonl", + str(spool), + "--command", + "bad/path", + ] + ) + assert invalid["returncode"] == 2, invalid + assert "command must use only" in "\n".join(invalid["payload"]["failures"]), invalid + assert len(read_jsonl(spool)) == 1, read_jsonl(spool) + + cli_emit = run( + [ + sys.executable, + str(YAO), + "telemetry-emit", + str(skill_dir), + "--output-jsonl", + str(spool), + "--event", + "skill_output", + "--activation-type", + "manual", + "--outcome", + "edited", + "--command", + "browser-plugin", + "--timestamp", + "2026-06-13T12:01:00Z", + ] + ) + assert cli_emit["ok"], cli_emit + assert cli_emit["payload"]["emitted"] is True, cli_emit + assert len(read_jsonl(spool)) == 2, read_jsonl(spool) + + imported = run( + [ + sys.executable, + str(YAO), + "telemetry-import", + str(skill_dir), + "--input-jsonl", + str(spool), + "--generated-at", + "2026-06-13T12:02:00Z", + ] + ) + assert imported["ok"], imported + assert imported["payload"]["imported_count"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["event_count"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 2, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-extension"] == 1, imported + assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-plugin"] == 1, imported + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 768315ca..96bae64d 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -74,6 +74,7 @@ def main() -> None: assert "quickstart" in parser_help, parser_help assert "review-studio" in parser_help, parser_help assert "telemetry-import" in parser_help, parser_help + assert "telemetry-emit" in parser_help, parser_help assert "--record-cli-telemetry" in parser_help, parser_help init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root))
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder