feat: import external telemetry events
This commit is contained in:
yaojingang
@@ -70,6 +70,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept
|
||||
- `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers.
|
||||
- `scripts/yao_cli_parser.py`: CLI argparse command surface, flags, choices, and command handler binding.
|
||||
- `scripts/yao_cli_telemetry.py`: opt-in metadata-only CLI run telemetry. Keep it free of prompt, argument, output, transcript, note, or message capture.
|
||||
- `scripts/import_telemetry_events.py`: external telemetry importer. Validate the whole input before appending events, and keep raw prompt/output/transcript/message/note fields blocked.
|
||||
|
||||
New helper modules that are imported by CLI/report scripts but are not standalone commands must declare:
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ PYTHON ?= python3
|
||||
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
|
||||
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
|
||||
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check telemetry-import-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
|
||||
eval:
|
||||
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
@@ -91,6 +91,9 @@ feedback-check:
|
||||
adoption-drift-check:
|
||||
$(PYTHON) tests/verify_adoption_drift.py
|
||||
|
||||
telemetry-import-check:
|
||||
$(PYTHON) tests/verify_telemetry_import.py
|
||||
|
||||
review-waivers-check:
|
||||
$(PYTHON) tests/verify_review_waivers.py
|
||||
|
||||
@@ -187,11 +190,11 @@ sync-local-install: package-check
|
||||
sync-active-install: package-check
|
||||
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
|
||||
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check telemetry-import-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
|
||||
ci-test:
|
||||
$(PYTHON) scripts/ci_test.py
|
||||
|
||||
clean:
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
||||
|
||||
@@ -31,7 +31,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
- a systems-thinking model that maps boundaries, feedback loops, drift risks, recurring failure patterns, and highest-leverage quality moves
|
||||
- three high-value next iteration directions after the first package is created
|
||||
- a lightweight feedback log that does not require a full promotion cycle
|
||||
- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates, with optional `yao.py` CLI run capture that records command names and outcomes without arguments or raw content
|
||||
- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates, with optional `yao.py` CLI run capture and external client JSONL import that record command names and outcomes without arguments or raw content
|
||||
- a Skill Atlas drift layer that reads aggregate adoption reports and surfaces portfolio-level drift signals without packaging raw telemetry logs
|
||||
- a baseline compare report for with-skill vs baseline review
|
||||
- a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits
|
||||
@@ -109,6 +109,7 @@ python3 scripts/yao.py system-model my-skill
|
||||
python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before adding scripts." --rating 4 --category boundary
|
||||
python3 scripts/yao.py adoption-drift my-skill --record-event skill_activation --activation-type explicit --outcome accepted
|
||||
YAO_CLI_TELEMETRY=1 python3 scripts/yao.py validate my-skill
|
||||
python3 scripts/yao.py telemetry-import my-skill --input-jsonl /tmp/external-client-events.jsonl --command browser-extension
|
||||
python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key trust-report --reviewer "Yao Team" --reason "Known warning accepted for this release with bounded follow-up." --expires-at 2026-09-30
|
||||
python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key permission-gates --reviewer "Yao Team" --reason "Permission warning accepted only for this non-governed release window." --expires-at 2026-09-30
|
||||
python3 scripts/yao.py review-annotations my-skill --add-annotation --gate-key output-lab --target-path reports/output_quality_scorecard.md --line 1 --body "Clarify recorded fixture vs model-executed evidence before release."
|
||||
@@ -414,6 +415,7 @@ Utility scripts that make the meta-skill operational:
|
||||
- `simulate_install.py`: extracts a generated zip into a temporary skill root and verifies entrypoint, manifest, interface, reports, and adapters can be loaded
|
||||
- `upgrade_check.py`: compares current and previous registry package metadata, recommends a version bump, and blocks incompatible upgrade claims
|
||||
- `render_adoption_drift_report.py`: records metadata-only local telemetry and renders adoption, missed-trigger, bad-output, script-error, and review-drift signals without packaging raw event logs
|
||||
- `import_telemetry_events.py`: imports external metadata-only telemetry JSONL after whole-file privacy validation, then refreshes the aggregate adoption drift report
|
||||
- `yao_cli_telemetry.py`: opt-in metadata-only `yao.py` run capture for command name, source, outcome, and failure class without command arguments or raw content
|
||||
- `render_review_waivers.py`: validates human reviewer risk approvals with gate keys, reasons, expiry dates, and blocker-safe waiver policy
|
||||
- `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain
|
||||
|
||||
@@ -61,6 +61,24 @@ python3 scripts/yao.py --no-cli-telemetry validate .
|
||||
|
||||
Successful CLI runs record `event=script_run`, `source=yao_cli`, `outcome=accepted`, and `failure_type=none`. Failed CLI runs record `outcome=failed` and `failure_type=script_error`. The command name is normalized to the subcommand only; command arguments are never recorded.
|
||||
|
||||
## External Client Import
|
||||
|
||||
External clients, browser extensions, editor adapters, or wrapper scripts may hand off already-sanitized JSONL through `telemetry-import`:
|
||||
|
||||
```bash
|
||||
python3 scripts/yao.py telemetry-import . \
|
||||
--input-jsonl /tmp/external-client-events.jsonl \
|
||||
--command browser-extension
|
||||
```
|
||||
|
||||
The importer defaults missing `source` to `external` and missing `command` to `external-client`. It validates the entire JSONL file before writing anything. If any line includes a raw content field, unsupported source, unsupported outcome, unsupported failure type, unknown field, malformed JSON, or an unsafe command name, the whole import is rejected and the existing local event stream is left untouched.
|
||||
|
||||
Use `--dry-run` to validate an external batch without writing `reports/telemetry_events.jsonl` or refreshing aggregate reports:
|
||||
|
||||
```bash
|
||||
python3 scripts/yao.py telemetry-import . --input-jsonl /tmp/external-client-events.jsonl --dry-run
|
||||
```
|
||||
|
||||
## Privacy Rule
|
||||
|
||||
The raw JSONL event log is local evidence and should not be distributed in skill packages. The distributable artifact is the aggregate report:
|
||||
@@ -79,7 +97,7 @@ Package builders should exclude `reports/telemetry_events.jsonl`. The root repos
|
||||
|
||||
## Iteration Loop
|
||||
|
||||
1. Capture metadata-only events locally, either manually with `adoption-drift --record-event` or automatically with opt-in `yao.py` CLI capture.
|
||||
1. Capture metadata-only events locally, either manually with `adoption-drift --record-event`, automatically with opt-in `yao.py` CLI capture, or through validated external JSONL import.
|
||||
2. Render `reports/adoption_drift_report.md`.
|
||||
3. Convert missed triggers into trigger eval cases.
|
||||
4. Convert bad outputs into Output Eval assertions and failure taxonomy entries.
|
||||
|
||||
+1
-1
@@ -16,7 +16,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2"
|
||||
"package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2",
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474"
|
||||
"package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9",
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -48,7 +48,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474",
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-13T13:30:00Z",
|
||||
"generated_at": "2026-06-13T13:45:00Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
|
||||
@@ -159,6 +159,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
@@ -265,7 +266,7 @@
|
||||
],
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
}
|
||||
@@ -371,7 +372,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -384,6 +385,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -456,14 +458,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -485,7 +487,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -506,7 +508,7 @@
|
||||
"strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -517,7 +519,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -598,7 +600,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -611,6 +613,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -683,14 +686,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -710,7 +713,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -731,7 +734,7 @@
|
||||
"strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -742,7 +745,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -958,6 +961,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
@@ -1064,7 +1068,7 @@
|
||||
],
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
}
|
||||
@@ -1170,7 +1174,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -1183,6 +1187,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -1255,14 +1260,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -1284,7 +1289,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -1305,7 +1310,7 @@
|
||||
"strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -1316,7 +1321,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -1397,7 +1402,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -1410,6 +1415,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -1482,14 +1488,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -1509,7 +1515,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -1530,7 +1536,7 @@
|
||||
"strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -1541,7 +1547,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -1757,6 +1763,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
@@ -1863,7 +1870,7 @@
|
||||
],
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
}
|
||||
@@ -1969,7 +1976,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -1982,6 +1989,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -2054,14 +2062,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -2083,7 +2091,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -2104,7 +2112,7 @@
|
||||
"strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -2114,7 +2122,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Expose script and permission metadata for downstream clients or installers to enforce.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -2189,7 +2197,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -2202,6 +2210,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -2274,14 +2283,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -2301,7 +2310,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -2322,7 +2331,7 @@
|
||||
"strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -2332,7 +2341,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Expose script and permission metadata for downstream clients or installers to enforce.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -2540,6 +2549,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
@@ -2646,7 +2656,7 @@
|
||||
],
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
}
|
||||
@@ -2752,7 +2762,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -2765,6 +2775,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -2837,14 +2848,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -2866,7 +2877,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -2887,7 +2898,7 @@
|
||||
"strategy": "Keep optional directories as relative resources next to SKILL.md.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -2898,7 +2909,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Scripts remain local optional resources and should advertise --help when executable.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -2972,7 +2983,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -2985,6 +2996,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -3057,14 +3069,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -3084,7 +3096,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -3105,7 +3117,7 @@
|
||||
"strategy": "Keep optional directories as relative resources next to SKILL.md.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -3116,7 +3128,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Scripts remain local optional resources and should advertise --help when executable.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -3323,6 +3335,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
@@ -3429,7 +3442,7 @@
|
||||
],
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
}
|
||||
@@ -3535,7 +3548,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -3548,6 +3561,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -3620,14 +3634,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -3649,7 +3663,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -3670,7 +3684,7 @@
|
||||
"strategy": "Install as project or user scoped skill source, preserving relative references and scripts.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -3681,7 +3695,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Scripts require workspace trust and operator/client approval outside this compiler.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
@@ -3759,7 +3773,7 @@
|
||||
},
|
||||
"file_write": {
|
||||
"required": true,
|
||||
"script_count": 47,
|
||||
"script_count": 48,
|
||||
"scripts": [
|
||||
"scripts/adjudicate_output_review.py",
|
||||
"scripts/build_confusion_matrix.py",
|
||||
@@ -3772,6 +3786,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
@@ -3844,14 +3859,14 @@
|
||||
},
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"checked_count": 60,
|
||||
"checked_count": 61,
|
||||
"failed_count": 0,
|
||||
"failed_scripts": []
|
||||
},
|
||||
"trust_summary": {
|
||||
"secret_findings": 0,
|
||||
"network_script_count": 3,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"subprocess_script_count": 7,
|
||||
"interactive_script_count": 0,
|
||||
"help_smoke_failed_count": 0
|
||||
@@ -3871,7 +3886,7 @@
|
||||
],
|
||||
"capability_counts": {
|
||||
"network": 3,
|
||||
"file_write": 47,
|
||||
"file_write": 48,
|
||||
"subprocess": 7,
|
||||
"interactive": 0
|
||||
},
|
||||
@@ -3892,7 +3907,7 @@
|
||||
"strategy": "Install as project or user scoped skill source, preserving relative references and scripts.",
|
||||
"counts": {
|
||||
"references": 32,
|
||||
"scripts": 70,
|
||||
"scripts": 71,
|
||||
"assets": 2,
|
||||
"reports": 41
|
||||
},
|
||||
@@ -3903,7 +3918,7 @@
|
||||
},
|
||||
"scripts": {
|
||||
"strategy": "Scripts require workspace trust and operator/client approval outside this compiler.",
|
||||
"script_count": 70,
|
||||
"script_count": 71,
|
||||
"help_smoke_failed_count": 0
|
||||
},
|
||||
"permissions": {
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 751,
|
||||
"other_text_tokens": 917960,
|
||||
"other_text_tokens": 922840,
|
||||
"estimated_initial_load_tokens": 944,
|
||||
"estimated_total_text_tokens": 918711,
|
||||
"relevant_file_count": 394,
|
||||
"estimated_total_text_tokens": 923591,
|
||||
"relevant_file_count": 398,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
"quality_density": 137.7
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 495,
|
||||
"archive_entry_count": 497,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 31.15,
|
||||
"duration_ms": 31.38,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 31.4,
|
||||
"duration_ms": 30.75,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 33.61,
|
||||
"duration_ms": 30.8,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 33.86,
|
||||
"duration_ms": 32.43,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 33.02,
|
||||
"duration_ms": 31.92,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 31.39,
|
||||
"duration_ms": 32.23,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 30.77,
|
||||
"duration_ms": 31.19,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 32.92,
|
||||
"duration_ms": 34.13,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 29.09,
|
||||
"duration_ms": 32.55,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.95,
|
||||
"duration_ms": 32.09,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 31.15 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 31.4 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 33.61 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 33.86 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 33.02 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 31.39 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 30.77 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 32.92 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 29.09 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.95 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 31.38 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 30.75 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 30.8 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 32.43 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 31.92 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 32.23 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 31.19 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 34.13 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 32.55 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 32.09 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474",
|
||||
"archive_entry_count": 495,
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c",
|
||||
"archive_entry_count": 497,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
- Package directory: `dist`
|
||||
- Targets: `4 / 4` adapters present
|
||||
- Archive present: `True`
|
||||
- Archive SHA256: `0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474`
|
||||
- Archive SHA256: `7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c`
|
||||
- Failures: `0`
|
||||
- Warnings: `0`
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2",
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474"
|
||||
"package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9",
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -53,7 +53,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474",
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -78,7 +78,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2"
|
||||
"package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
- Maturity: `governed`
|
||||
- Owner: `Yao Team`
|
||||
- License: `MIT`
|
||||
- Package SHA256: `495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2`
|
||||
- Archive SHA256: `0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474`
|
||||
- Package SHA256: `57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9`
|
||||
- Archive SHA256: `7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c`
|
||||
- Install simulated: `True`
|
||||
|
||||
## Compatibility
|
||||
|
||||
@@ -242,12 +242,12 @@
|
||||
|
||||
<section>
|
||||
<h2>核心指标</h2>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native enforcement targets</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>70 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>495 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native enforcement targets</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>71 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>497 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>审查闸门</h2>
|
||||
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 70 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
|
||||
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 71 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
|
||||
</section>
|
||||
|
||||
<div class="twocol">
|
||||
@@ -313,7 +313,7 @@
|
||||
</section>
|
||||
|
||||
<section id="trust" class="twocol">
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>70</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>71</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
|
||||
</section>
|
||||
|
||||
@@ -344,12 +344,12 @@
|
||||
|
||||
<section id="registry" class="twocol">
|
||||
<div class="panel"><h2>注册审计</h2><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section id="release" class="twocol">
|
||||
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>495</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474</code></dd></div></dl></div>
|
||||
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>497</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c</code></dd></div></dl></div>
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
|
||||
+159
-118
File diff suppressed because it is too large
Load Diff
@@ -2,8 +2,8 @@
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"scanned_files": 154,
|
||||
"script_count": 70,
|
||||
"scanned_files": 155,
|
||||
"script_count": 71,
|
||||
"internal_module_count": 10,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
@@ -12,18 +12,18 @@
|
||||
"network_script_count": 3,
|
||||
"network_policy_covered_count": 3,
|
||||
"network_policy_missing_count": 0,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"permission_required_count": 3,
|
||||
"permission_approved_count": 3,
|
||||
"permission_missing_count": 0,
|
||||
"permission_invalid_count": 0,
|
||||
"permission_expired_count": 0,
|
||||
"help_smoke_checked_count": 60,
|
||||
"help_smoke_checked_count": 61,
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 154,
|
||||
"package_sha256": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2"
|
||||
"package_hash_file_count": 155,
|
||||
"package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
@@ -234,6 +234,20 @@
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/import_telemetry_events.py",
|
||||
"interface": "cli",
|
||||
"interface_declared": false,
|
||||
"interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.",
|
||||
"has_argparse": true,
|
||||
"has_main_guard": true,
|
||||
"uses_input": false,
|
||||
"uses_network": false,
|
||||
"uses_file_write": true,
|
||||
"uses_subprocess": false,
|
||||
"network_urls": [],
|
||||
"network_hosts": []
|
||||
},
|
||||
{
|
||||
"path": "scripts/init_skill.py",
|
||||
"interface": "cli",
|
||||
@@ -1057,9 +1071,9 @@
|
||||
"help_smoke": {
|
||||
"enabled": true,
|
||||
"timeout_seconds": 5.0,
|
||||
"candidate_count": 60,
|
||||
"checked_count": 60,
|
||||
"passed_count": 60,
|
||||
"candidate_count": 61,
|
||||
"checked_count": 61,
|
||||
"passed_count": 61,
|
||||
"failed_count": 0,
|
||||
"skipped_count": 10,
|
||||
"failed_scripts": [],
|
||||
@@ -1204,6 +1218,16 @@
|
||||
"stdout_excerpt": "usage: governance_check.py [-h] [--require-manifest] skill_dir\n\nCheck skill governance metadata and lifecycle readiness.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit\n --require-manifest",
|
||||
"stderr_excerpt": ""
|
||||
},
|
||||
{
|
||||
"path": "scripts/import_telemetry_events.py",
|
||||
"command": "python3 scripts/import_telemetry_events.py --help",
|
||||
"returncode": 0,
|
||||
"timed_out": false,
|
||||
"passed": true,
|
||||
"has_help_text": true,
|
||||
"stdout_excerpt": "usage: import_telemetry_events.py [-h] --input-jsonl INPUT_JSONL\n [--events-jsonl EVENTS_JSONL]\n [--output-json OUTPUT_JSON]\n [--output-md OU",
|
||||
"stderr_excerpt": ""
|
||||
},
|
||||
{
|
||||
"path": "scripts/init_skill.py",
|
||||
"command": "python3 scripts/init_skill.py --help",
|
||||
@@ -1771,6 +1795,7 @@
|
||||
"scripts/cross_packager.py",
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/probe_runtime_permissions.py",
|
||||
"scripts/promotion_checker.py",
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
# Security Trust Report
|
||||
|
||||
- OK: `True`
|
||||
- Scanned files: `154`
|
||||
- Scripts: `70`
|
||||
- Scanned files: `155`
|
||||
- Scripts: `71`
|
||||
- Internal script modules: `10`
|
||||
- Secret findings: `0`
|
||||
- Network-capable scripts: `3`
|
||||
- Network policy covered scripts: `3`
|
||||
- Network policy missing scripts: `0`
|
||||
- File-write scripts: `47`
|
||||
- File-write scripts: `48`
|
||||
- Permission approvals: `3 / 3`
|
||||
- Permission approval gaps: `0`
|
||||
- CLI help smoke checked: `60`
|
||||
- CLI help smoke checked: `61`
|
||||
- CLI help smoke failures: `0`
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `154`
|
||||
- Package SHA256: `495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2`
|
||||
- Package hash files: `155`
|
||||
- Package SHA256: `57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9`
|
||||
|
||||
## Failures
|
||||
|
||||
@@ -54,8 +54,8 @@
|
||||
|
||||
- Enabled: `True`
|
||||
- Timeout seconds: `5.0`
|
||||
- Checked scripts: `60`
|
||||
- Passed scripts: `60`
|
||||
- Checked scripts: `61`
|
||||
- Passed scripts: `61`
|
||||
- Failed scripts: `none`
|
||||
|
||||
## Script Surface
|
||||
@@ -76,6 +76,7 @@
|
||||
| scripts/export_skill_ir.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/github_benchmark_scan.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/governance_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/import_telemetry_events.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/init_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/judge_blind_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
| scripts/lint_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
|
||||
|
||||
@@ -23,6 +23,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
|
||||
- Upgrade Check v0 for registry package diffs, semver bump recommendations, breaking-change notes, and release evidence.
|
||||
- Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, iteration candidate generation, and Skill Atlas drift-signal input.
|
||||
- CLI Telemetry Capture v0 for opt-in `yao.py` command-run metadata that records command name, source, outcome, failure class, and timestamp without command arguments or raw user/model content.
|
||||
- External Telemetry Import v0 for whole-file validated metadata-only JSONL ingestion from non-CLI clients, with dry-run support and all-or-nothing rejection on raw content fields or schema violations.
|
||||
- Review Waivers v0 for human warning acceptance with reviewer, reason, scope, expiry, and blocker-safe policy.
|
||||
- Governed Permission Gates v0 for reviewer-approved network, file-write, and subprocess capabilities with scope, reason, expiry, evidence, and target-enforcement mapping.
|
||||
- Runtime Permission Probes v0 for packaged target adapter checks, explicit native-enforcement flags, metadata fallback evidence, and residual permission risks.
|
||||
@@ -34,7 +35,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
|
||||
- Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case.
|
||||
- Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue.
|
||||
|
||||
This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, external client telemetry capture, provider-native installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open.
|
||||
This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, installer-level permission coverage is now locally enforced during install simulation and local install sync, opt-in `yao.py` CLI telemetry can capture metadata-only real run signals, validated external JSONL imports can bring non-CLI client signals into the same drift loop, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, native client telemetry hooks, provider-native installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open.
|
||||
|
||||
## Coverage Matrix
|
||||
|
||||
@@ -55,7 +56,7 @@ This is still not the final world-class state. Target-native behavior contracts
|
||||
| Install Simulation | `scripts/simulate_install.py`, `reports/install_simulation.md`, `tests/verify_install_simulation.py` with installer permission coverage checks | v0 landed |
|
||||
| Local Install Sync Preflight | `scripts/sync_local_install.py`, `tests/verify_local_install_sync.py`, `Makefile` package-check prerequisites | v0 landed |
|
||||
| Upgrade Check | `scripts/upgrade_check.py`, `reports/upgrade_check.md`, `tests/verify_upgrade_check.py` | v0 landed |
|
||||
| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `scripts/yao_cli_telemetry.py`, `reports/adoption_drift_report.md`, `tests/verify_adoption_drift.py`, `tests/verify_yao_cli.py`, `references/telemetry-drift-method.md` | v0 landed |
|
||||
| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `scripts/yao_cli_telemetry.py`, `scripts/import_telemetry_events.py`, `reports/adoption_drift_report.md`, `tests/verify_adoption_drift.py`, `tests/verify_telemetry_import.py`, `tests/verify_yao_cli.py`, `references/telemetry-drift-method.md` | v0 landed |
|
||||
| Review Waivers | `scripts/render_review_waivers.py`, `reports/review_waivers.md`, `tests/verify_review_waivers.py`, `references/review-waiver-method.md` | v0 landed |
|
||||
| Governed Permission Gates | `security/permission_policy.json`, `scripts/trust_check.py`, `scripts/render_review_studio.py`, `tests/verify_trust_check.py`, `tests/verify_review_studio.py` | v0 landed |
|
||||
| Runtime Permission Probes | `scripts/probe_runtime_permissions.py`, `reports/runtime_permission_probes.md`, `tests/verify_runtime_permission_probes.py`, `tests/verify_review_studio.py` | v0 landed |
|
||||
@@ -85,9 +86,9 @@ Next move: add richer source-line anchors inside generated reports and record re
|
||||
|
||||
### 4. Multi-skill operation now links Atlas with drift, but needs real client capture
|
||||
|
||||
The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, aggregate drift signals, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, feed next iteration candidates into Review Studio, and publish aggregate drift input for Atlas. `yao.py` now adds opt-in automatic CLI run capture through `YAO_CLI_TELEMETRY=1` or `--record-cli-telemetry`, recording only `source=yao_cli`, normalized subcommand name, outcome, failure class, and timestamp. It still needs external client-side capture beyond the local CLI.
|
||||
The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, aggregate drift signals, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, feed next iteration candidates into Review Studio, and publish aggregate drift input for Atlas. `yao.py` now adds opt-in automatic CLI run capture through `YAO_CLI_TELEMETRY=1` or `--record-cli-telemetry`, recording only `source=yao_cli`, normalized subcommand name, outcome, failure class, and timestamp. `telemetry-import` now accepts already-sanitized external client JSONL, validates the whole file before appending, supports dry-run, defaults `source=external`, and rejects any raw content or unknown metadata fields without mutating the local event stream. It still needs native client hooks that produce those external JSONL batches automatically.
|
||||
|
||||
Next move: connect non-CLI client activations and failure history so Atlas can rank stale, drifting, or conflicting skills by real usage impact.
|
||||
Next move: wire native client hooks to emit the external metadata-only JSONL batches automatically, then let Atlas rank stale, drifting, or conflicting skills by real usage impact.
|
||||
|
||||
### 5. Trust report is structural, not full security review
|
||||
|
||||
@@ -102,22 +103,22 @@ Next move: add real client or installer permission enforcement integration.
|
||||
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden |
|
||||
| Runtime Conformance | `5 / 5` targets passing |
|
||||
| Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts |
|
||||
| Trust | `0` secret findings, `1` pinned dependency file, `9` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `60 / 60` CLI help smoke checks passing across `69` scripts, source-contract hash scope explicit |
|
||||
| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `61 / 61` CLI help smoke checks passing across `71` scripts, source-contract hash scope explicit |
|
||||
| Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired |
|
||||
| Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility |
|
||||
| Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility |
|
||||
| Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix |
|
||||
| Package Verification | `4 / 4` target adapters present, archive verified, `495` zip entries, `0` failures, `0` warnings |
|
||||
| Install Simulation | archive with `495` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings |
|
||||
| Package Verification | `4 / 4` target adapters present, archive verified, `497` zip entries, `0` failures, `0` warnings |
|
||||
| Install Simulation | archive with `497` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings |
|
||||
| Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures |
|
||||
| Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes |
|
||||
| Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture is available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages |
|
||||
| Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages |
|
||||
| Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0 |
|
||||
| Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers |
|
||||
| Review Studio | decision `review`, world-class score `92`, `13` gates, `0` blockers, `2` warnings, `2` review actions, `0` open annotation blockers |
|
||||
| IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable |
|
||||
| Context Budget | initial load `944/1000`, under the production budget |
|
||||
| CI | `make ci-test` target count is `56` after the VS Code package target update |
|
||||
| CI | `make ci-test` target count is `57` after the telemetry import gate update |
|
||||
|
||||
## Next Highest-Leverage Moves
|
||||
|
||||
@@ -125,4 +126,4 @@ Next move: add real client or installer permission enforcement integration.
|
||||
2. Add native client or installer runtime enforcement for approved high-permission capabilities.
|
||||
3. Run the new provider-backed output runner against holdout cases with real credentials, then record the current blind A/B decisions before claiming fully ready status.
|
||||
4. Add real reviewer annotation records during the next human review pass.
|
||||
5. Connect external client telemetry capture and failure history so Atlas drift signals include non-CLI live usage, not only manual samples or opt-in CLI runs.
|
||||
5. Wire native client telemetry hooks so external JSONL batches are produced automatically, not only imported from already-sanitized files.
|
||||
|
||||
@@ -609,7 +609,7 @@
|
||||
<ul class="list"><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">Skill 作者、复用团队和后续 reviewer。</span><span data-lang="en">Skill 作者、复用团队和后续 reviewer。</span></li><li><span data-lang="zh-CN">创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。</span><span data-lang="en">创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。</span></li></ul>
|
||||
</aside>
|
||||
</div>
|
||||
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14859 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14859 个词/字。</span></small></article></div>
|
||||
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14978 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14978 个词/字。</span></small></article></div>
|
||||
</section>
|
||||
|
||||
<section id="overview">
|
||||
@@ -641,10 +641,10 @@
|
||||
<article class="panel metrics-note">
|
||||
<h3><span data-lang="zh-CN">指标判读</span><span data-lang="en">Reading</span></h3>
|
||||
<p><span data-lang="zh-CN">先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。</span><span data-lang="en">Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.</span></p>
|
||||
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14859 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14859 个词/字。</span></small></li></ol>
|
||||
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14978 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14978 个词/字。</span></small></li></ol>
|
||||
</article>
|
||||
</div>
|
||||
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在，便于人工阅读。</span><span data-lang="en">README.md 已存在，便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在，便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在，便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">compiled_targets.json 已存在。</span><span data-lang="en">compiled_targets.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制，可维护性较好。</span><span data-lang="en">入口文件保持克制，可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 350 个词/字，references 约 14859 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14859 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高，建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高，建议压缩入口或拆分 references。</span></li></ul></div></article></div>
|
||||
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在，便于人工阅读。</span><span data-lang="en">README.md 已存在，便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在，便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在，便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">compiled_targets.json 已存在。</span><span data-lang="en">compiled_targets.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制，可维护性较好。</span><span data-lang="en">入口文件保持克制，可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 350 个词/字，references 约 14978 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14978 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高，建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高，建议压缩入口或拆分 references。</span></li></ul></div></article></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -720,7 +720,7 @@
|
||||
<div>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th><th><span data-lang="zh-CN">证据</span><span data-lang="en">Evidence</span></th><th><span data-lang="zh-CN">建议</span><span data-lang="en">Action</span></th></tr></thead>
|
||||
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简，并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span><span data-lang="en">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强：入口约 350 个词/字，references 约 14859 个词/字。</span><span data-lang="en">上下文成本需要补强：入口约 350 个词/字，references 约 14859 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
|
||||
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简，并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span><span data-lang="en">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强：入口约 350 个词/字，references 约 14978 个词/字。</span><span data-lang="en">上下文成本需要补强：入口约 350 个词/字，references 约 14978 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
|
||||
</table>
|
||||
<div class="two-col quality-panels">
|
||||
<article class="panel">
|
||||
@@ -771,7 +771,7 @@
|
||||
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
|
||||
</div>
|
||||
<div class="two-col">
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-0.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-1.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-2.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="30.2 69.8" stroke-dashoffset="-3.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="66.0 34.0" stroke-dashoffset="-34.0" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">305项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-0.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-1.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-2.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="29.9 70.1" stroke-dashoffset="-3.7" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="66.4 33.6" stroke-dashoffset="-33.6" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">306项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
|
||||
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
|
||||
|
||||
+20
-20
@@ -96,7 +96,7 @@
|
||||
"label": "上下文成本",
|
||||
"score": 42,
|
||||
"reasons": [
|
||||
"入口约 350 个词/字，references 约 14859 个词/字。",
|
||||
"入口约 350 个词/字，references 约 14978 个词/字。",
|
||||
"分数越高代表上下文成本越低。",
|
||||
"上下文成本偏高，建议压缩入口或拆分 references。"
|
||||
]
|
||||
@@ -200,7 +200,7 @@
|
||||
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。"
|
||||
],
|
||||
"gaps": [
|
||||
"上下文成本需要补强：入口约 350 个词/字，references 约 14859 个词/字。"
|
||||
"上下文成本需要补强：入口约 350 个词/字，references 约 14978 个词/字。"
|
||||
],
|
||||
"recommendations": [
|
||||
"先改触发边界，再扩展工作流。",
|
||||
@@ -433,7 +433,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 70
|
||||
"file_count": 71
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -448,7 +448,7 @@
|
||||
"file_count": 171
|
||||
}
|
||||
],
|
||||
"file_count": 305,
|
||||
"file_count": 306,
|
||||
"folder_count": 4,
|
||||
"distribution": [
|
||||
{
|
||||
@@ -473,7 +473,7 @@
|
||||
},
|
||||
{
|
||||
"label": "scripts",
|
||||
"value": 70
|
||||
"value": 71
|
||||
},
|
||||
{
|
||||
"label": "evals",
|
||||
@@ -602,7 +602,7 @@
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 70
|
||||
"file_count": 71
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
@@ -948,8 +948,8 @@
|
||||
"failures": []
|
||||
},
|
||||
"trust_security": {
|
||||
"scanned_files": 154,
|
||||
"script_count": 70,
|
||||
"scanned_files": 155,
|
||||
"script_count": 71,
|
||||
"internal_module_count": 10,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
@@ -958,18 +958,18 @@
|
||||
"network_script_count": 3,
|
||||
"network_policy_covered_count": 3,
|
||||
"network_policy_missing_count": 0,
|
||||
"file_write_script_count": 47,
|
||||
"file_write_script_count": 48,
|
||||
"permission_required_count": 3,
|
||||
"permission_approved_count": 3,
|
||||
"permission_missing_count": 0,
|
||||
"permission_invalid_count": 0,
|
||||
"permission_expired_count": 0,
|
||||
"help_smoke_checked_count": 60,
|
||||
"help_smoke_checked_count": 61,
|
||||
"help_smoke_failed_count": 0,
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 154,
|
||||
"package_sha256": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2"
|
||||
"package_hash_file_count": 155,
|
||||
"package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1007,8 +1007,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2",
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474"
|
||||
"package_sha256": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9",
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1039,7 +1039,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474",
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1055,8 +1055,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474",
|
||||
"archive_entry_count": 495,
|
||||
"archive_sha256": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c",
|
||||
"archive_entry_count": 497,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -1067,7 +1067,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 495,
|
||||
"archive_entry_count": 497,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -1134,12 +1134,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474"
|
||||
"to": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2"
|
||||
"to": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -78,6 +78,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
|
||||
@@ -70,12 +70,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "0f80e2e194d003a99da34f73923b9459f260b2d177c5f1f6ec7104fdd5713474"
|
||||
"to": "7ae19fa1196580440ddebf5ad124810f724f945cb76ac3a4a519da9edde2f14c"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "495c5824cf84b56ef77bebc75e415191425e2f1131f0a4027d17037631b52fa2"
|
||||
"to": "57991433fa63fd3a5b34cf9bad76ab118895476dbe12810929c8a396b73796c9"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -50,6 +50,7 @@ DEFAULT_TARGETS = [
|
||||
"review-studio-check",
|
||||
"feedback-check",
|
||||
"adoption-drift-check",
|
||||
"telemetry-import-check",
|
||||
"review-waivers-check",
|
||||
"review-annotations-check",
|
||||
"baseline-compare-check",
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from render_adoption_drift_report import display_path, normalize_event, render_report, skill_defaults
|
||||
|
||||
|
||||
def load_candidate_events(
|
||||
input_jsonl: Path,
|
||||
defaults: dict[str, str],
|
||||
default_source: str,
|
||||
default_command: str,
|
||||
) -> tuple[list[dict[str, str]], list[str]]:
|
||||
events: list[dict[str, str]] = []
|
||||
failures: list[str] = []
|
||||
for index, line in enumerate(input_jsonl.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
raw = json.loads(line)
|
||||
except json.JSONDecodeError as exc:
|
||||
failures.append(f"line {index}: invalid JSONL event: {exc.msg}")
|
||||
continue
|
||||
if not isinstance(raw, dict):
|
||||
failures.append(f"line {index}: telemetry event must be a JSON object")
|
||||
continue
|
||||
raw.setdefault("source", default_source)
|
||||
raw.setdefault("command", default_command)
|
||||
event, event_failures = normalize_event(raw, defaults, f"line {index}")
|
||||
failures.extend(event_failures)
|
||||
if event:
|
||||
events.append(event)
|
||||
return events, failures
|
||||
|
||||
|
||||
def append_events(events_jsonl: Path, events: list[dict[str, str]]) -> None:
|
||||
events_jsonl.parent.mkdir(parents=True, exist_ok=True)
|
||||
with events_jsonl.open("a", encoding="utf-8") as handle:
|
||||
for event in events:
|
||||
handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n")
|
||||
|
||||
|
||||
def import_events(
|
||||
skill_dir: Path,
|
||||
input_jsonl: Path,
|
||||
events_jsonl: Path | None = None,
|
||||
output_json: Path | None = None,
|
||||
output_md: Path | None = None,
|
||||
generated_at: str | None = None,
|
||||
default_source: str = "external",
|
||||
default_command: str = "external-client",
|
||||
dry_run: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
input_jsonl = input_jsonl.resolve()
|
||||
reports_dir = skill_dir / "reports"
|
||||
events_jsonl = (events_jsonl or reports_dir / "telemetry_events.jsonl").resolve()
|
||||
output_json = (output_json or reports_dir / "adoption_drift_report.json").resolve()
|
||||
output_md = (output_md or reports_dir / "adoption_drift_report.md").resolve()
|
||||
failures: list[str] = []
|
||||
if not input_jsonl.exists():
|
||||
failures.append(f"Input telemetry JSONL does not exist: {display_path(input_jsonl)}")
|
||||
candidate_events: list[dict[str, str]] = []
|
||||
else:
|
||||
candidate_events, load_failures = load_candidate_events(
|
||||
input_jsonl,
|
||||
skill_defaults(skill_dir),
|
||||
default_source,
|
||||
default_command,
|
||||
)
|
||||
failures.extend(load_failures)
|
||||
|
||||
adoption_report: dict[str, Any] | None = None
|
||||
if not failures and not dry_run:
|
||||
append_events(events_jsonl, candidate_events)
|
||||
adoption_report = render_report(
|
||||
skill_dir,
|
||||
events_jsonl=events_jsonl,
|
||||
output_json=output_json,
|
||||
output_md=output_md,
|
||||
generated_at=generated_at,
|
||||
)
|
||||
|
||||
return {
|
||||
"ok": not failures,
|
||||
"schema_version": "1.0",
|
||||
"skill_dir": display_path(skill_dir),
|
||||
"input_jsonl": display_path(input_jsonl),
|
||||
"events_jsonl": display_path(events_jsonl),
|
||||
"dry_run": dry_run,
|
||||
"imported_count": 0 if failures or dry_run else len(candidate_events),
|
||||
"candidate_count": len(candidate_events),
|
||||
"failures": failures,
|
||||
"imported_preview": candidate_events[:5],
|
||||
"artifacts": {
|
||||
"events_jsonl": display_path(events_jsonl),
|
||||
"adoption_drift_json": display_path(output_json),
|
||||
"adoption_drift_md": display_path(output_md),
|
||||
},
|
||||
"adoption_drift": {
|
||||
"summary": adoption_report.get("summary", {}) if adoption_report else {},
|
||||
"artifacts": adoption_report.get("artifacts", {}) if adoption_report else {},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Import external metadata-only telemetry into a local skill event stream.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--input-jsonl", required=True)
|
||||
parser.add_argument("--events-jsonl")
|
||||
parser.add_argument("--output-json")
|
||||
parser.add_argument("--output-md")
|
||||
parser.add_argument("--generated-at")
|
||||
parser.add_argument("--source", choices=["external", "manual", "unknown", "yao_cli"], default="external")
|
||||
parser.add_argument("--command", default="external-client")
|
||||
parser.add_argument("--dry-run", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
report = import_events(
|
||||
Path(args.skill_dir),
|
||||
Path(args.input_jsonl),
|
||||
events_jsonl=Path(args.events_jsonl) if args.events_jsonl else None,
|
||||
output_json=Path(args.output_json) if args.output_json else None,
|
||||
output_md=Path(args.output_md) if args.output_md else None,
|
||||
generated_at=args.generated_at,
|
||||
default_source=args.source,
|
||||
default_command=args.command,
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -705,6 +705,28 @@ def command_adoption_drift(args: argparse.Namespace) -> int:
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_telemetry_import(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir, "--input-jsonl", args.input_jsonl]
|
||||
if args.events_jsonl:
|
||||
cmd.extend(["--events-jsonl", args.events_jsonl])
|
||||
if args.output_json:
|
||||
cmd.extend(["--output-json", args.output_json])
|
||||
if args.output_md:
|
||||
cmd.extend(["--output-md", args.output_md])
|
||||
if args.generated_at:
|
||||
cmd.extend(["--generated-at", args.generated_at])
|
||||
if args.source:
|
||||
cmd.extend(["--source", args.source])
|
||||
if args.telemetry_command:
|
||||
cmd.extend(["--command", args.telemetry_command])
|
||||
if args.dry_run:
|
||||
cmd.append("--dry-run")
|
||||
result = run_script("import_telemetry_events.py", cmd)
|
||||
print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_review_waivers(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir]
|
||||
|
||||
@@ -284,6 +284,21 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int]
|
||||
adoption_drift_cmd.add_argument("--version")
|
||||
adoption_drift_cmd.set_defaults(func=_handler(command_handlers, "command_adoption_drift"))
|
||||
|
||||
telemetry_import_cmd = subparsers.add_parser(
|
||||
"telemetry-import",
|
||||
help="Import external metadata-only telemetry JSONL and refresh the adoption drift report.",
|
||||
)
|
||||
telemetry_import_cmd.add_argument("skill_dir", nargs="?", default=".")
|
||||
telemetry_import_cmd.add_argument("--input-jsonl", required=True)
|
||||
telemetry_import_cmd.add_argument("--events-jsonl")
|
||||
telemetry_import_cmd.add_argument("--output-json")
|
||||
telemetry_import_cmd.add_argument("--output-md")
|
||||
telemetry_import_cmd.add_argument("--generated-at")
|
||||
telemetry_import_cmd.add_argument("--source", choices=["external", "manual", "unknown", "yao_cli"], default="external")
|
||||
telemetry_import_cmd.add_argument("--command", dest="telemetry_command", default="external-client")
|
||||
telemetry_import_cmd.add_argument("--dry-run", action="store_true")
|
||||
telemetry_import_cmd.set_defaults(func=_handler(command_handlers, "command_telemetry_import"))
|
||||
|
||||
review_waivers_cmd = subparsers.add_parser(
|
||||
"review-waivers",
|
||||
help="Render or update human reviewer waiver evidence for Review Studio.",
|
||||
|
||||
@@ -122,6 +122,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
|
||||
@@ -34,6 +34,7 @@
|
||||
"scripts/export_skill_ir.py",
|
||||
"scripts/github_benchmark_scan.py",
|
||||
"scripts/governance_check.py",
|
||||
"scripts/import_telemetry_events.py",
|
||||
"scripts/init_skill.py",
|
||||
"scripts/judge_blind_eval.py",
|
||||
"scripts/lint_skill.py",
|
||||
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
TMP = ROOT / "tests" / "tmp_telemetry_import"
|
||||
IMPORTER = ROOT / "scripts" / "import_telemetry_events.py"
|
||||
YAO = ROOT / "scripts" / "yao.py"
|
||||
|
||||
|
||||
def run(cmd: list[str]) -> dict:
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
||||
payload = json.loads(proc.stdout) if proc.stdout.strip() else {}
|
||||
return {
|
||||
"ok": proc.returncode == 0,
|
||||
"returncode": proc.returncode,
|
||||
"payload": payload,
|
||||
"stdout": proc.stdout,
|
||||
"stderr": proc.stderr,
|
||||
}
|
||||
|
||||
|
||||
def write_jsonl(path: Path, rows: list[dict]) -> None:
|
||||
path.write_text("\n".join(json.dumps(row, ensure_ascii=False) for row in rows) + "\n", encoding="utf-8")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if TMP.exists():
|
||||
shutil.rmtree(TMP)
|
||||
TMP.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / "scripts" / "init_skill.py"),
|
||||
"external-telemetry-demo",
|
||||
"--description",
|
||||
"Import external client telemetry into a reusable metadata-only drift loop.",
|
||||
"--output-dir",
|
||||
str(TMP),
|
||||
],
|
||||
cwd=ROOT,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
skill_dir = TMP / "external-telemetry-demo"
|
||||
input_jsonl = TMP / "external-events.jsonl"
|
||||
write_jsonl(
|
||||
input_jsonl,
|
||||
[
|
||||
{
|
||||
"event": "skill_activation",
|
||||
"activation_type": "explicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"timestamp": "2026-06-13T11:00:00Z",
|
||||
},
|
||||
{
|
||||
"event": "script_run",
|
||||
"activation_type": "manual",
|
||||
"outcome": "failed",
|
||||
"failure_type": "script_error",
|
||||
"timestamp": "2026-06-13T11:01:00Z",
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
dry_run = run(
|
||||
[
|
||||
sys.executable,
|
||||
str(IMPORTER),
|
||||
str(skill_dir),
|
||||
"--input-jsonl",
|
||||
str(input_jsonl),
|
||||
"--command",
|
||||
"chrome-extension",
|
||||
"--dry-run",
|
||||
]
|
||||
)
|
||||
assert dry_run["ok"], dry_run
|
||||
assert dry_run["payload"]["dry_run"] is True, dry_run
|
||||
assert dry_run["payload"]["candidate_count"] == 2, dry_run
|
||||
assert dry_run["payload"]["imported_count"] == 0, dry_run
|
||||
assert not (skill_dir / "reports" / "telemetry_events.jsonl").exists(), dry_run
|
||||
|
||||
imported = run(
|
||||
[
|
||||
sys.executable,
|
||||
str(IMPORTER),
|
||||
str(skill_dir),
|
||||
"--input-jsonl",
|
||||
str(input_jsonl),
|
||||
"--command",
|
||||
"chrome-extension",
|
||||
"--generated-at",
|
||||
"2026-06-13T11:02:00Z",
|
||||
]
|
||||
)
|
||||
assert imported["ok"], imported
|
||||
assert imported["payload"]["imported_count"] == 2, imported
|
||||
assert imported["payload"]["adoption_drift"]["summary"]["event_count"] == 2, imported
|
||||
assert imported["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 2, imported
|
||||
assert imported["payload"]["adoption_drift"]["summary"]["command_counts"]["chrome-extension"] == 2, imported
|
||||
events = [
|
||||
json.loads(line)
|
||||
for line in (skill_dir / "reports" / "telemetry_events.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
]
|
||||
assert len(events) == 2, events
|
||||
assert {event["source"] for event in events} == {"external"}, events
|
||||
assert {event["command"] for event in events} == {"chrome-extension"}, events
|
||||
markdown = (skill_dir / "reports" / "adoption_drift_report.md").read_text(encoding="utf-8")
|
||||
assert "source=`external` command=`chrome-extension`" in markdown, markdown
|
||||
|
||||
unsafe_jsonl = TMP / "unsafe-events.jsonl"
|
||||
write_jsonl(
|
||||
unsafe_jsonl,
|
||||
[
|
||||
{
|
||||
"event": "skill_activation",
|
||||
"activation_type": "explicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"timestamp": "2026-06-13T11:03:00Z",
|
||||
},
|
||||
{
|
||||
"event": "skill_activation",
|
||||
"activation_type": "explicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"prompt": "raw user prompt must never be imported",
|
||||
"timestamp": "2026-06-13T11:04:00Z",
|
||||
},
|
||||
],
|
||||
)
|
||||
unsafe = run([sys.executable, str(IMPORTER), str(skill_dir), "--input-jsonl", str(unsafe_jsonl)])
|
||||
assert unsafe["returncode"] == 2, unsafe
|
||||
assert not unsafe["payload"]["ok"], unsafe
|
||||
assert "raw content fields" in "\n".join(unsafe["payload"]["failures"]), unsafe
|
||||
events_after_failure = (skill_dir / "reports" / "telemetry_events.jsonl").read_text(encoding="utf-8").splitlines()
|
||||
assert len(events_after_failure) == 2, events_after_failure
|
||||
|
||||
cli_input_jsonl = TMP / "cli-external-events.jsonl"
|
||||
write_jsonl(
|
||||
cli_input_jsonl,
|
||||
[
|
||||
{
|
||||
"event": "skill_output",
|
||||
"activation_type": "manual",
|
||||
"outcome": "edited",
|
||||
"failure_type": "none",
|
||||
"timestamp": "2026-06-13T11:05:00Z",
|
||||
}
|
||||
],
|
||||
)
|
||||
cli = run(
|
||||
[
|
||||
sys.executable,
|
||||
str(YAO),
|
||||
"telemetry-import",
|
||||
str(skill_dir),
|
||||
"--input-jsonl",
|
||||
str(cli_input_jsonl),
|
||||
"--command",
|
||||
"browser-plugin",
|
||||
"--generated-at",
|
||||
"2026-06-13T11:06:00Z",
|
||||
]
|
||||
)
|
||||
assert cli["ok"], cli
|
||||
assert cli["payload"]["imported_count"] == 1, cli
|
||||
assert cli["payload"]["adoption_drift"]["summary"]["event_count"] == 3, cli
|
||||
assert cli["payload"]["adoption_drift"]["summary"]["source_types"]["external"] == 3, cli
|
||||
assert cli["payload"]["adoption_drift"]["summary"]["command_counts"]["browser-plugin"] == 1, cli
|
||||
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -73,6 +73,7 @@ def main() -> None:
|
||||
parser_help = yao_cli_module.build_parser().format_help()
|
||||
assert "quickstart" in parser_help, parser_help
|
||||
assert "review-studio" in parser_help, parser_help
|
||||
assert "telemetry-import" in parser_help, parser_help
|
||||
assert "--record-cli-telemetry" in parser_help, parser_help
|
||||
|
||||
init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root))
|
||||
|
||||
Reference in New Issue
Block a user