frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Upgrade meta skill governance reports

Browse Source
This commit is contained in:
yaojingang
2026-06-13 18:00:32 +08:00
parent 30fb0bf6d4
commit 3fb08802e9
134 changed files with 25466 additions and 1122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -21,6 +21,7 @@ tests/tmp_system_model/
tests/tmp_security/
tests/tmp_baseline_compare.*
reports/release_snapshots/
reports/telemetry_events.jsonl
.yao/
# Local business-skill experiments belong outside this meta-skill repo unless promoted intentionally.
Makefile
+36 -3
View File
@@ -2,7 +2,7 @@ PYTHON ?= python3
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
eval:
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
@@ -43,18 +43,42 @@ skill-report-charts-check:
skill-ir-check:
$(PYTHON) tests/verify_skill_ir.py
compiler-check:
$(PYTHON) tests/verify_compile_skill.py
output-eval-check:
$(PYTHON) tests/verify_output_eval_lab.py
output-execution-check:
$(PYTHON) tests/verify_output_execution_runs.py
output-review-adjudication-check:
$(PYTHON) tests/verify_output_review_adjudication.py
runtime-conformance-check:
$(PYTHON) tests/verify_conformance_suite.py
runtime-permission-check:
$(PYTHON) tests/verify_runtime_permission_probes.py
trust-check:
$(PYTHON) tests/verify_trust_check.py
skill-atlas-check:
$(PYTHON) tests/verify_skill_atlas.py
registry-audit-check:
$(PYTHON) tests/verify_registry_audit.py
package-verify-check:
$(PYTHON) tests/verify_package_verification.py
install-simulation-check:
$(PYTHON) tests/verify_install_simulation.py
upgrade-check:
$(PYTHON) tests/verify_upgrade_check.py
review-viewer-check:
$(PYTHON) tests/verify_review_viewer.py
@@ -64,6 +88,15 @@ review-studio-check:
feedback-check:
$(PYTHON) tests/verify_feedback.py
adoption-drift-check:
$(PYTHON) tests/verify_adoption_drift.py
review-waivers-check:
$(PYTHON) tests/verify_review_waivers.py
review-annotations-check:
$(PYTHON) tests/verify_review_annotations.py
baseline-compare-check:
$(PYTHON) tests/verify_baseline_compare.py
@@ -154,11 +187,11 @@ sync-local-install:
sync-active-install:
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
ci-test:
$(PYTHON) scripts/ci_test.py
clean:
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_output_eval tests/tmp_conformance tests/tmp_trust tests/tmp_skill_atlas tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
find . -type d -name __pycache__ -prune -exec rm -rf {} +
README.md
+55 -10
View File
@@ -22,16 +22,20 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
- a front-loaded intent dialogue with an intent confidence gate, so the system keeps clarifying when the true job, outputs, exclusions, or standards are still fuzzy
- a silent-by-default GitHub benchmark scan plus reference synthesis that studies top public repositories and world-class pattern tracks, then surfaces only real conflicts or uncertainty to the user
- a generated visual HTML overview for each newly initialized skill
- a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, and release evidence
- a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, adoption drift, reviewer waivers, reviewer annotations, release evidence, and per-warning fix actions
- Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports
- a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks
- a side-by-side HTML review studio for first-pass human review
- an artifact design profile that defines visual direction, layout patterns, and quality gates for reports, tutorials, dashboards, screenshots, and review pages
- a prompt quality profile that abstracts need modeling, RTF mapping, complexity, and quality checks into reviewer-visible evidence instead of bloating `SKILL.md`
- a systems-thinking model that maps boundaries, feedback loops, drift risks, recurring failure patterns, and highest-leverage quality moves
- three high-value next iteration directions after the first package is created
- a lightweight feedback log that does not require a full promotion cycle
- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates
- a baseline compare report for with-skill vs baseline review
- a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits
- Skill IR as the platform-neutral semantic contract plus client-specific adapters
- Skill IR as the platform-neutral semantic contract, plus compiler reports and client-specific adapters
- Registry audit metadata with package version, owner, license, checksum, and compatibility matrix
- governance, promotion, and portability checks built into the default flow
## Architecture
@@ -85,7 +89,7 @@ Weighted score formula: `sum(score / 10 * weight)`.
2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit.
3. Let `quickstart` clarify intent first, then run silent benchmark scan and reference synthesis; it only surfaces explicit questions when intent is still unclear or when there is a real design conflict.
4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode.
5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/review-studio.html` to inspect release blockers and evidence paths in one page before adding more structure.
5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/review-studio.html` to inspect release blockers, permission approvals, and evidence paths in one page before adding more structure.
Or use the unified authoring CLI:
@@ -102,14 +106,26 @@ python3 scripts/yao.py artifact-design-profile my-skill
python3 scripts/yao.py prompt-quality-profile my-skill
python3 scripts/yao.py system-model my-skill
python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before adding scripts." --rating 4 --category boundary
python3 scripts/yao.py adoption-drift my-skill --record-event skill_activation --activation-type explicit --outcome accepted
python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key trust-report --reviewer "Yao Team" --reason "Known warning accepted for this release with bounded follow-up." --expires-at 2026-09-30
python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key permission-gates --reviewer "Yao Team" --reason "Permission warning accepted only for this non-governed release window." --expires-at 2026-09-30
python3 scripts/yao.py review-annotations my-skill --add-annotation --gate-key output-lab --target-path reports/output_quality_scorecard.md --line 1 --body "Clarify recorded fixture vs model-executed evidence before release."
python3 scripts/yao.py baseline-compare
python3 scripts/yao.py check-update
python3 scripts/yao.py package . --platform generic --output-dir dist
python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json
python3 scripts/yao.py compile-skill . --target openai --target claude --target generic
python3 scripts/yao.py package . --platform generic --output-dir dist
python3 scripts/yao.py output-eval
python3 scripts/yao.py output-exec
python3 scripts/yao.py output-review
python3 scripts/yao.py conformance .
python3 scripts/yao.py trust .
python3 scripts/yao.py runtime-permissions . --package-dir dist
python3 scripts/yao.py skill-atlas --workspace-root .
python3 scripts/yao.py registry-audit .
python3 scripts/yao.py package-verify . --package-dir dist --require-zip
python3 scripts/yao.py install-simulate . --package-dir dist
python3 scripts/yao.py upgrade-check . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json
```
## Local Development Source
@@ -160,7 +176,9 @@ python3 scripts/judge_blind_eval.py --description-file SKILL.md --cases evals/bl
python3 scripts/context_sizer.py .
python3 scripts/resource_boundary_check.py .
python3 scripts/governance_check.py . --require-manifest
python3 scripts/compile_skill.py .
python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip
python3 scripts/probe_runtime_permissions.py . --package-dir dist
python3 tests/verify_packager_failures.py
```
@@ -179,7 +197,11 @@ python3 scripts/yao.py workspace-flow --target root --label first-pass
python3 scripts/yao.py review-viewer my-skill
python3 scripts/yao.py review --target root
python3 scripts/yao.py release-snapshot --target root --label release-candidate
python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json
python3 scripts/yao.py compile-skill .
python3 scripts/yao.py package . --platform openai --platform claude --platform generic --output-dir dist --zip
python3 scripts/yao.py runtime-permissions . --package-dir dist
python3 scripts/yao.py package-verify . --package-dir dist --require-zip
python3 scripts/yao.py test
```
@@ -225,7 +247,9 @@ The homepage panel below is generated from the current eval suite so the family-
Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md)
<!-- END:EVAL_RESULTS -->
- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks and carry IR provenance plus semantic parity metadata
- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks and carry IR provenance, semantic parity metadata, and target-native behavior contracts
- target compiler validation: `openai`, `claude`, `generic`, and Agent Skills compatible contracts are compiled from Skill IR with generated-file mappings, adapter modes, native surfaces, permission enforcement notes, and unsupported-feature notes
- runtime permission probes: `openai`, `claude`, and `generic` adapters expose explicit permission contracts; current targets report `0` native-enforcement adapters and `3` metadata fallbacks with residual risks visible to reviewers
- portability score: `100/100` with neutral activation, execution, trust, and degradation metadata preserved across all exported targets
- description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately
- judge-backed blind eval: root, team frontend review, and governed incident command now pass an independent rubric judge on blind holdout prompts
@@ -234,8 +258,8 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa
- governance and resource-boundary checks are part of the default test path
- root governance maturity score: `90/100`; governed benchmark example: `95/100`
- CJK-aware trigger matching is now covered by explicit Chinese build, packaging, eval, and near-neighbor cases
- context budgets: root `994/1000`, complex benchmark `790/1000`, governed benchmark `760/1000`
- quality density: root `130.8`, complex benchmark `164.6`, governed benchmark `171.1`
- context budgets: root `987/1000`, complex benchmark `790/1000`, governed benchmark `760/1000`
- quality density: root `131.7`, complex benchmark `164.6`, governed benchmark `171.1`
- regression milestones are tracked in [reports/regression_history.md](reports/regression_history.md)
- description drift history is tracked in [reports/description_drift_history.md](reports/description_drift_history.md)
- route confusion is tracked in [reports/route_scorecard.md](reports/route_scorecard.md)
@@ -244,6 +268,14 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa
- candidate lifecycle states are published in [reports/candidate_registry.md](reports/candidate_registry.md)
- lightweight with-skill vs baseline comparison is published in [reports/baseline-compare.md](reports/baseline-compare.md)
- Review Studio 2.0 gate evidence is published in [reports/review-studio.html](reports/review-studio.html)
- Review Studio fix actions are embedded in [reports/review-studio.json](reports/review-studio.json)
- reviewer waiver evidence is published in [reports/review_waivers.md](reports/review_waivers.md)
- target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md)
- registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md)
- package archive verification is published in [reports/package_verification.md](reports/package_verification.md)
- temporary local install simulation is published in [reports/install_simulation.md](reports/install_simulation.md)
- upgrade diff, version-bump recommendation, and release-note evidence are published in [reports/upgrade_check.md](reports/upgrade_check.md)
- local-first adoption and drift telemetry is summarized in [reports/adoption_drift_report.md](reports/adoption_drift_report.md)
- context budget summaries are tracked in [reports/context_budget.md](reports/context_budget.md)
- portability status is tracked in [reports/portability_score.md](reports/portability_score.md)
@@ -353,7 +385,7 @@ Utility scripts that make the meta-skill operational:
- `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries
- `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs
- `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence
- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, package, and test as one entrypoint
- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint
- `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report
- `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot
- `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger
@@ -366,10 +398,21 @@ Utility scripts that make the meta-skill operational:
- `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage
- `render_skill_overview.py`: generates the white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, top-right language switch, v2 scorecard, inline SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap
- `export_skill_ir.py`: exports the 2.0 platform-neutral Skill IR contract from `SKILL.md`, manifest, interface metadata, evals, resources, and reports
- `run_output_eval.py`: runs the Output Eval Lab v0 with static with-skill vs baseline assertion grading and writes `reports/output_quality_scorecard.*`
- `compile_skill.py`: compiles Skill IR into target-specific semantic contracts, generated-file maps, adapter modes, target-native behavior contracts, preserved semantics, warnings, and unsupported-feature notes
- `run_output_eval.py`: runs the Output Eval Lab v0 with static with-skill vs baseline assertion grading, blind A/B review pack generation, and separate answer key artifacts
- `run_output_execution.py`: records output-eval execution evidence, distinguishing recorded fixtures, command runners, and provider-backed model runs with timing and token metadata
- `local_output_eval_runner.py`: deterministic local runner for command-executed output-eval smoke evidence without claiming provider-backed model generation
- `adjudicate_output_review.py`: records reviewer choices for blind A/B output evals, compares them with the answer key, and renders pending, match, disagreement, and invalid-decision audit reports
- `render_review_annotations.py`: records reviewer annotations tied to Review Studio gates, source/report paths, and optional line numbers, with open blocker annotations reflected in Review Studio decisions
- `run_conformance_suite.py`: verifies runtime conformance for OpenAI, Claude, Agent Skills, VS Code/Copilot-style, and generic targets
- `trust_check.py`: generates the trust/security report for scripts, dependencies, secret risk, trust metadata, and package integrity
- `trust_check.py`: generates the trust/security report for scripts, dependencies, secret risk, bounded network host policy, execution-level `--help` smoke checks, permission inputs, trust metadata, and stable source-contract integrity
- `build_skill_atlas.py`: builds the Skill Atlas catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview for a multi-skill workspace
- `registry_audit.py`: builds registry package metadata and audits version, owner, license, checksum, Skill IR source, and compatibility matrix
- `verify_package.py`: verifies generated package manifests, target adapters, zip archive safety, archive checksum, and registry parity
- `simulate_install.py`: extracts a generated zip into a temporary skill root and verifies entrypoint, manifest, interface, reports, and adapters can be loaded
- `upgrade_check.py`: compares current and previous registry package metadata, recommends a version bump, and blocks incompatible upgrade claims
- `render_adoption_drift_report.py`: records metadata-only local telemetry and renders adoption, missed-trigger, bad-output, script-error, and review-drift signals without packaging raw event logs
- `render_review_waivers.py`: validates human reviewer risk approvals with gate keys, reasons, expiry dates, and blocker-safe waiver policy
- `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain
- `check_update.py`: checks GitHub for a newer `VERSION` or remote manifest version and reports a reinstall hint without modifying local files
- `render_output_risk_profile.py`: predicts output-specific failure modes such as generic headings, citation clutter, screenshot mistakes, weak Markdown tables, and missing execution assumptions
@@ -444,6 +487,8 @@ The typical flow is:
Examples:
```bash
python3 scripts/export_skill_ir.py ./yao-meta-skill --output-json ./yao-meta-skill/reports/skill-ir.json
python3 scripts/compile_skill.py ./yao-meta-skill --target openai --target claude --target generic
python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --expectations evals/packaging_expectations.json --zip
python3 scripts/context_sizer.py ./yao-meta-skill
python3 scripts/resource_boundary_check.py ./yao-meta-skill
SKILL.md
+2 -2
View File
@@ -32,11 +32,11 @@ Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes](
6. Add only folders and gates that earn their keep.
7. Surface the top three next iteration directions.
Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Review Studio](references/review-studio-method.md).
Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Registry](references/distribution-registry-method.md), [Telemetry](references/telemetry-drift-method.md), [Waivers](references/review-waiver-method.md), [Review Studio](references/review-studio-method.md).
## Skill OS 2.0 Gates
For production, library, governed, or team-distributed work, use Skill IR, trigger + output eval, Skill Atlas, runtime conformance, trust report, and Review Studio before release.
For production, library, governed, or team-distributed work, use Skill IR, target compiler contracts, trigger + output eval, Skill Atlas, runtime conformance, trust report, registry metadata, package verification, install simulation, upgrade check, adoption/drift telemetry, reviewer waiver evidence, and Review Studio before release.
## First-Turn Style
evals/README.md
+27 -1
View File
@@ -15,8 +15,15 @@ Contents:
- `improved_description.txt`: current stronger trigger description
- `../reports/description_optimization*.{json,md}`: generated route-optimization reports for the root skill
- `failure-cases.md`: current weak spots and regression targets
- `packaging_expectations.json`: required packaging behaviors for supported targets, including IR provenance and semantic parity fields
- `packaging_expectations.json`: required packaging behaviors for supported targets, including IR provenance, compiler contracts, and semantic parity fields
- `output/`: Output Eval Lab cases, fixtures, and schema for with-skill vs baseline assertion grading
- registry audit tests verify package metadata, compatibility, checksum, owner, license, and Skill IR provenance
- package verification tests verify generated manifests, target adapters, zip archive safety, archive checksum, and registry parity
- install simulation tests verify archive extraction, entrypoint loading, manifest/interface readability, adapter loading, and unsafe zip blocking
- upgrade check tests verify version-bump recommendations, target diff detection, compatibility regression blocking, and release-note evidence
- adoption drift tests verify metadata-only telemetry aggregation, privacy-field blocking, and exclusion of raw local telemetry logs from zip packages
- review waiver tests verify reviewer, reason, expiry, active coverage, expired records, invalid records, and Review Studio integration
- compiler tests verify Skill IR target contracts, adapter modes, generated-file mappings, unsupported targets, and markdown/json report artifacts
- `../reports/`: generated suite JSON plus the homepage-visible family summary panel source
Use:
@@ -34,10 +41,24 @@ python3 scripts/render_iteration_ledger.py
python3 tests/verify_description_optimization.py
python3 tests/verify_route_confusion.py
python3 tests/verify_failure_regressions.py
python3 scripts/compile_skill.py . --target openai --target claude --target generic
python3 tests/verify_compile_skill.py
python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip
python3 scripts/verify_package.py . --package-dir dist --require-zip
python3 tests/verify_packager_failures.py
python3 tests/verify_package_verification.py
python3 scripts/simulate_install.py . --package-dir dist
python3 tests/verify_install_simulation.py
python3 scripts/run_output_eval.py
python3 tests/verify_output_eval_lab.py
python3 scripts/registry_audit.py .
python3 tests/verify_registry_audit.py
python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json
python3 tests/verify_upgrade_check.py
python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted
python3 tests/verify_adoption_drift.py
python3 scripts/render_review_waivers.py .
python3 tests/verify_review_waivers.py
```
Regression scope now includes:
@@ -58,3 +79,8 @@ Regression scope now includes:
- calibration summaries that surface score gaps, threshold margins, and risk bands for each acceptance gate
- family-level drift history that records which blind and adversarial families stay clean over time
- output-quality scorecards that compare without-skill and with-skill artifacts through assertion grading, including near-neighbor and file-backed governed cases
- install-simulation checks that prove generated archives can be extracted into a temporary local skill root and loaded without touching the active global skills directory
- upgrade-readiness checks that compare registry package baselines, recommend semver bumps, and block breaking changes without an adequate declared version
- local-first adoption drift checks that turn accepted, edited, missed, rejected, script-error, and review-overdue metadata into next iteration candidates without storing raw prompts or outputs
- reviewer-waiver checks that keep warning acceptance explicit, expiring, and separate from non-waivable blockers
- compiler checks that turn Skill IR into reviewable target contracts before packaging embeds those contracts into adapters
evals/output/schema.json
+21 -1
View File
@@ -2,7 +2,7 @@
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://yao.example/output-eval-case.schema.json",
"title": "Yao Output Eval Case",
"description": "Static v0 output-eval case for comparing without-skill and with-skill outputs.",
"description": "Output-eval case for comparing without-skill and with-skill outputs, with optional execution metadata.",
"type": "object",
"additionalProperties": false,
"required": ["id", "prompt", "baseline_output", "with_skill_output", "assertions"],
@@ -57,6 +57,26 @@
"type": "object",
"additionalProperties": true
},
"execution": {
"type": "object",
"description": "Optional case-level execution metadata for fixture, command, or provider-backed output runs.",
"additionalProperties": true,
"properties": {
"mode": {
"type": "string",
"enum": ["recorded_fixture", "command", "model"]
},
"provider": {
"type": "string"
},
"model": {
"type": "string"
},
"holdout": {
"type": "boolean"
}
}
},
"metadata": {
"type": "object",
"additionalProperties": true
evals/packaging_expectations.json
+6
View File
@@ -12,6 +12,12 @@
"ir_schema_version",
"semantic_contract",
"semantic_parity",
"compiler",
"compiled_contract",
"permission_contract",
"target_permission_contract",
"target_native_contract",
"target_transform",
"canonical_metadata",
"canonical_format",
"activation_mode",
references/distribution-registry-method.md
+57
View File
@@ -0,0 +1,57 @@
# Distribution Registry Method
Registry metadata turns a local skill directory into an installable and reviewable package record.
## When To Use
Use registry audit for library, governed, team-distributed, or upgrade-sensitive skills. Scaffold skills can skip it until reuse is real.
## Required Evidence
- package name, version, owner, maturity, review cadence, and license
- Skill IR source and schema version
- trust level and package SHA256
- package verification status and archive SHA256 when a zip distribution is built
- install simulation status when an archive is meant to be installed locally or by a team
- adoption/drift aggregate status when local telemetry exists
- target compatibility matrix
- upgrade diff, recommended version bump, and breaking-change notes when a previous package baseline exists
- links to overview, Review Studio, trust report, conformance matrix, package verification, install simulation, adoption drift evidence, and review waivers
## Release Rule
Do not publish a team package when registry audit reports missing version, hash, owner, review cadence, license, valid Skill IR, or passing compatibility for declared targets.
For an installable archive, first build the distribution and run package verification:
```bash
python3 scripts/yao.py package . --platform openai --platform claude --platform generic --output-dir dist --zip
python3 scripts/yao.py package-verify . --package-dir dist --require-zip
python3 scripts/yao.py install-simulate . --package-dir dist
python3 scripts/yao.py registry-audit .
python3 scripts/yao.py upgrade-check . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json
```
Do not claim archive readiness when package verification reports unsafe zip paths, missing target adapters, missing package manifest, registry metadata mismatch, or unreadable archive contents.
Do not claim install readiness when install simulation cannot extract the archive into a temporary skill root, load `SKILL.md` frontmatter, read `manifest.json`, read `agents/interface.yaml`, find the overview and Review Studio reports, or load each generated adapter.
Do not include raw `reports/telemetry_events.jsonl` in a distributed package. Include only aggregate adoption drift reports, and block release review when telemetry contains raw prompts, outputs, transcripts, notes, or messages.
Review waiver evidence may be distributed as `reports/review_waivers.md/json` because it is metadata-only reviewer accountability. Do not store raw prompts, outputs, transcripts, credentials, or private customer detail in waiver reasons.
Do not claim upgrade readiness when upgrade check reports an insufficient version bump, target removal without a major bump, compatibility regression without a major bump, package name change, or invalid semver. Include `reports/upgrade_check.md` in the reviewer evidence bundle so release notes and migration guidance are tied to the exact registry diff.
## Reviewer Gate
A reviewer should be able to answer:
1. Which package version is being installed?
2. Who owns the package?
3. Which targets are compatible?
4. Which checksum identifies the reviewed package contents?
5. Which reports prove trust and runtime readiness?
6. Was the installable archive verified, and which checksum identifies it?
7. Was the archive install-simulated in a temporary local skill root?
8. What changed since the previous package, and does the declared version bump match the recommended bump?
9. Are adoption and drift signals summarized without packaging raw local telemetry?
references/output-eval-method.md
+53
View File
@@ -38,10 +38,63 @@ The first v0 scorecard reports:
- with-skill pass rate
- absolute delta
- failed assertions and failure taxonomy
- execution mode, timing, and token evidence when `reports/output_execution_runs.md` is generated
- blind A/B review pack count
- recommended next fixes
Production promotion should require the with-skill pass rate to beat baseline and should explain every failed assertion.
## Execution Evidence
Run execution evidence after the scorecard:
```bash
python3 scripts/yao.py output-exec
```
By default, this records the current case outputs as `recorded_fixture`. That is useful for reproducibility, but it is not model-executed evidence. To collect real run evidence, pass `--runner-command` with a command or JSON string list. The runner receives a JSON request on stdin and should return JSON with:
- `output`
- optional `execution_kind`: `command` or `model`
- optional `provider` and `model`
- optional `usage.input_tokens`, `usage.output_tokens`, and `usage.total_tokens`
Only runs that return provider/model metadata or `execution_kind: "model"` should count as model-executed. If token usage is absent, the report may estimate tokens, but the estimate must be labeled as estimated.
For local release-gate smoke evidence without external model credentials, use the deterministic runner:
```bash
python3 scripts/yao.py output-exec --runner-command '["python3","scripts/local_output_eval_runner.py"]'
```
This verifies the command-runner contract, timing capture, grading path, and failure handling. It must not be described as provider-backed model evidence.
## Blind A/B Review
Every output eval run should also generate:
- `reports/output_blind_review_pack.md`
- `reports/output_blind_review_pack.json`
- `reports/output_blind_answer_key.json`
The review pack must hide whether Variant A or Variant B came from the baseline or the skill-guided output. The answer key is separate audit evidence and should only be opened after a reviewer has made a judgment.
## Reviewer Adjudication
After blind review, record reviewer choices in `reports/output_review_decisions.json` and run:
```bash
python3 scripts/adjudicate_output_review.py --write-template
python3 scripts/yao.py output-review
```
The adjudication report writes:
- `reports/output_review_adjudication.json`
- `reports/output_review_adjudication.md`
When no reviewer decisions exist, the report should say the cases are pending. Do not count pending cases as human agreement. Only a real `winner_variant` of `A` or `B` should contribute to agreement rate, disagreement count, and reviewer judgment count.
## Anti-Overfitting
Keep a small public smoke set and a separate holdout set. Rotate real failures into the taxonomy instead of editing only the prompt that failed. Add near-neighbor cases whenever the output looks good but the boundary is still unclear.
references/packaging-contracts.md
+18 -5
View File
@@ -1,6 +1,6 @@
# Packaging Contracts
`cross_packager.py` is not just an export helper. It defines and validates platform contracts.
`cross_packager.py` is not just an export helper. It validates platform contracts and embeds target compiler output from `compile_skill.py`.
## Current Targets
@@ -15,8 +15,13 @@ Each target contract defines:
- required output fields
- required output files
- field mapping from the neutral source metadata
- compiled contract from Skill IR
- target transform metadata, including generated files and unsupported features
- portable execution metadata
- trust-boundary metadata
- permission contract metadata from the trust report
- target-specific permission representation and reviewer notes
- target-native behavior contract for native surface, activation policy, resource strategy, script strategy, permission enforcement, install scope, review artifacts, and fallback behavior
- degradation strategy metadata
## Failure Handling
@@ -27,6 +32,8 @@ When `--expectations` is provided:
- missing required fields cause exit code `2`
- validation failures are emitted in the JSON report
After packaging, run `scripts/probe_runtime_permissions.py` against the generated package directory. Packaging creates the permission metadata; the runtime permission probe verifies that each target adapter exposes the contract, target-specific representation, native-enforcement flag, operator note, and residual metadata-fallback risk.
## Source Of Truth
The platform-neutral semantic source is Skill IR when it exists:
@@ -39,10 +46,13 @@ The structural validation sources remain:
- `SKILL.md`
- `agents/interface.yaml`
Target-specific metadata is generated at packaging time. The adapter must carry
`ir_source`, `ir_schema_version`, `job_to_be_done`, `semantic_contract`, and
`semantic_parity` so reviewers can see whether the target preserved the core
skill meaning or fell back to frontmatter-only metadata.
Target-specific metadata is generated through `scripts/compile_skill.py` and
then embedded at packaging time. The adapter must carry `compiler`,
`compiled_contract`, `permission_contract`, `target_permission_contract`,
`target_native_contract`, `target_transform`, `ir_source`, `ir_schema_version`,
`job_to_be_done`, `semantic_contract`, and `semantic_parity` so reviewers can
see whether the target preserved the core skill meaning or fell back to
frontmatter-only metadata.
## Portability Model
@@ -51,7 +61,10 @@ The packaging layer now preserves four portable semantics from the neutral sourc
- activation
- execution
- trust
- permissions
- degradation
- platform-neutral skill meaning from Skill IR
- target-specific native behavior notes for activation, resources, scripts, permission enforcement, install scope, review artifacts, and fallback behavior
- target-specific compile notes for generated files, adapter mode, preserved semantics, and unsupported features
This means portability is not just "can it export a file?" but also "does the exported target preserve the source package's activation and safety assumptions?"
references/platform-capability-matrix.md
+16 -9
View File
@@ -2,17 +2,20 @@
This matrix describes the current packaging targets and their support level.
| Target | Metadata Adapter | Output Contract | Snapshot Test | Portability Semantics | Notes |
| --- | --- | --- | --- | --- | --- |
| `openai` | Yes | Yes | Yes | activation, execution, trust, degradation | Generates `targets/openai/agents/openai.yaml` |
| `claude` | Yes | Yes | Yes | activation, execution, trust, degradation | Generates `targets/claude/README.md` plus adapter metadata |
| `generic` | Yes | Yes | Yes | activation, execution, trust, degradation | Uses neutral adapter metadata only |
| Target | Metadata Adapter | Compiler Contract | Native Behavior Contract | Output Contract | Snapshot Test | Portability Semantics | Notes |
| --- | --- | --- | --- | --- | --- | --- | --- |
| `openai` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Generates `targets/openai/agents/openai.yaml` |
| `claude` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Generates `targets/claude/README.md` plus adapter metadata |
| `generic` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Uses neutral adapter metadata only |
| `agent-skills-compatible` | Neutral source | Yes | Yes | Source-compatible | Yes | activation, execution, trust, permissions, degradation, native behavior | Keeps canonical `SKILL.md` plus `agents/interface.yaml` source shape |
## Current Support Model
- `openai`: strongest metadata adapter support
- `claude`: lightweight compatibility adapter, behavior still relies mainly on neutral source files
- `generic`: lowest-friction export for neutral Agent Skills consumers
- `openai`: strongest metadata adapter support with an explicit compiler contract.
- `claude`: lightweight compatibility adapter with an explicit compiler contract and fallback notes.
- `generic`: lowest-friction export for neutral Agent Skills consumers.
- `agent-skills-compatible`: canonical source shape with compiler evidence for review and distribution.
- runtime permission probes currently report metadata fallback for generated targets; no target is claimed as native-enforced until a client or installer integration can actually enforce the permission model.
## Portable Semantics
@@ -21,15 +24,19 @@ Each target now preserves:
- activation mode and optional path filters
- execution context and shell choice
- trust tier and remote inline-execution policy
- permission contract for network, file-write, subprocess, and interactive script surfaces
- target-native behavior contract for native surface, activation policy, resource strategy, script strategy, permission enforcement, install scope, review artifacts, and fallback behavior
- degradation strategy for unsupported client behavior
- generated-file mapping and adapter mode from `reports/compiled_targets.json`
## Explicit Non-Goals
This project does not yet implement:
- runtime-specific behavior transforms
- client SDK integration
- provider-specific execution logic
- provider-native installer actions or account-level activation changes
- native runtime permission enforcement
## Degradation Rule
references/review-studio-method.md
+60 -4
View File
@@ -6,19 +6,27 @@ Review Studio is the release-facing audit surface for a skill package. It does n
- Show release blockers and warnings before the package deepens.
- Link every gate back to a concrete evidence artifact.
- Generate review actions for every blocker and warning, with source-fix location and verification command.
- Make human warning acceptance auditable through a waiver ledger.
- Make reviewer comments auditable through an annotation ledger tied to gates, source/report paths, and optional line numbers.
- Keep review flow vertical: summary first, gates second, supporting details after.
- Avoid hiding output quality, runtime, trust, and portfolio issues across separate pages.
- Avoid hiding output quality, runtime, trust, portfolio, and operating-loop issues across separate pages.
## Required Gates
1. Intent Canvas: intent confidence and unresolved input/output/exclusion gaps.
2. Trigger Lab: route scorecard, misroutes, ambiguous cases, and near-neighbor safety.
3. Output Lab: with-skill vs baseline delta, case count, file-backed cases, near-neighbor cases, and boundary cases.
3. Output Lab: with-skill vs baseline delta, execution mode, timing/token evidence, case count, file-backed cases, near-neighbor cases, boundary cases, blind A/B review pack evidence, and reviewer adjudication status.
4. Context Budget: initial load, budget tier, warnings, and quality density.
5. Runtime Matrix: target conformance pass/fail and degradation notes.
6. Trust Report: secret scan, script surface, dependency pinning, network/interactive flags, and package hash.
7. Skill Atlas: route collisions, stale skills, owner gaps, and no-route opportunities.
8. Release Notes: promotion status, migration notes, known gaps, and next move.
7. Permission Gates: reviewer-approved capability scope, reason, expiry, evidence, and target-enforcement notes.
8. Runtime Permission Probes: packaged adapter permission contracts, native-enforcement flags, metadata fallback notes, and residual risks.
9. Skill Atlas: route collisions, stale skills, owner gaps, and no-route opportunities.
10. Operations Loop: local-first metadata-only adoption, missed-trigger, bad-output, script-error, and review-drift signals.
11. Review Waivers: human risk approvals, active coverage, expired approvals, invalid records, and expiry policy.
12. Registry Audit: package metadata, install evidence, compatibility entries, and archive/source checksums.
13. Release Notes: promotion status, migration notes, known gaps, and next move.
## Gate Semantics
@@ -27,3 +35,51 @@ Review Studio is the release-facing audit surface for a skill package. It does n
- `block`: do not claim production, library, governed, or public readiness until fixed.
For library and governed skills, Output Lab should have at least five cases and cover file-backed, near-neighbor, and boundary scenarios.
Production, library, and governed reviews should also show a blind A/B review pack. The Review Studio gate may warn when scorecard evidence exists but no blind pack is present, because the package can prove assertions but not yet reduce reviewer bias.
When `reports/output_execution_runs.json` exists, Review Studio should show the number of variant runs, command-executed runs, model-executed runs, recorded fixtures, timing-observed runs, and token-estimated runs. Recorded fixtures are valid reproducibility evidence, but they must not be described as model-executed output evidence.
When `reports/output_review_adjudication.json` exists, Review Studio should show reviewed pairs and pending pairs. Pending reviewer decisions are acceptable as an explicit state, but they must not be counted as agreement or human review evidence. Invalid adjudication records should block release because they make the blind review audit untrustworthy.
The Operations Loop must never display raw telemetry logs. It should link only to `reports/adoption_drift_report.md`; privacy or schema violations are blockers.
The Runtime Permission Probes gate is evaluated after packaging, because it reads generated target adapters. A missing probe can warn in lighter modes, but governed release should not claim target permission readiness without `reports/runtime_permission_probes.json`.
The Review Waivers gate must never convert a blocker into a pass. Waivers only cover warning-level risks, require reviewer, reason, scope, and expiry, and must link only to `reports/review_waivers.md`.
Review Annotations are not waivers. They are reviewer comments attached to a Review Studio gate plus a relative source/report path and optional line number. Use them to preserve review context, requested edits, and source-line notes. Open blocker annotations should make the Review Studio decision `blocked` until the annotation is resolved or deferred with reviewer rationale. Open warning annotations can move the package into review, but they do not create gate-specific `review_actions`; actions remain reserved for non-pass gates.
## Review Actions
Every non-pass gate must produce a `review_actions` entry in `reports/review-studio.json`. When all gates pass, `review_actions` should be an empty list and the page should explicitly say there are no blocker or warning actions.
Each action must include:
- `gate_key`
- `status`
- `summary`
- `why`
- `source_fix`
- `source_refs`
- `evidence`
- `verification_command`
`source_refs` must be structured entries with relative `path`, human label, kind, existence flag, best-effort line number, and relative link when the file exists. They should point to the smallest useful report or source file, not just a broad directory.
The HTML page should render these actions before the detailed supporting sections so a reviewer can move directly from warning to fix. Action entries do not change gate count or score; they make the current decision more operational.
## Review Annotations
`reports/review_annotations.json` is the structured ledger, and `reports/review_annotations.md` is the human-readable review note surface. Each annotation should include:
- `gate_key`
- `target_path`
- `line` when a useful source line exists
- `severity`
- `status`
- `reviewer`
- `body`
- optional `suggested_action`
The ledger should reject absolute paths or paths that escape the skill directory. Missing target files are allowed as visible evidence gaps, not hidden failures.
references/review-waiver-method.md
+57
View File
@@ -0,0 +1,57 @@
# Review Waiver Method
Review waivers make human risk acceptance explicit. They are not a way to hide problems; they are a local audit record for warning-level risks that the reviewer intentionally accepts for a bounded release window.
## When To Use
Use a waiver when:
- Review Studio shows a warning that is understood and intentionally accepted.
- The warning cannot be fixed before release without a worse tradeoff.
- A reviewer can name the reason, scope, evidence, and expiry date.
Do not use a waiver for blocker gates in v0. Blockers must be fixed before production, library, governed, or public readiness is claimed. In governed mode, missing or invalid high-permission approvals are blockers and should be fixed in `security/permission_policy.json`, not waived.
## Required Fields
Every waiver must include:
- `gate_key`: the Review Studio gate being accepted.
- `decision`: `accepted-risk`, `false-positive`, or `temporary-exception`.
- `reviewer`: the accountable human or team.
- `reason`: a concrete reason of at least 20 characters.
- `created_at`: ISO date.
- `expires_at`: ISO date.
- `evidence`: optional path or note that explains the decision.
- `scope`: default `current-release`.
## Release Semantics
- Invalid waiver records block Review Studio.
- Expired waiver records stay visible and no longer cover warnings.
- Active waivers cover only the exact gate key they name.
- A warning without an active waiver remains visible as a warning.
- Raw user prompts, outputs, credentials, and private transcripts must not be stored in waiver reasons.
## Commands
Render or validate the ledger:
```bash
python3 scripts/render_review_waivers.py .
```
Add a bounded approval:
```bash
python3 scripts/yao.py review-waivers . \
--add-waiver \
--gate-key trust-report \
--reviewer "Yao Team" \
--reason "Network-capable scripts are documented and bounded for this release." \
--expires-at 2026-09-30
```
For a non-governed release where `permission-gates` is only a warning, the same command can name `--gate-key permission-gates`. Governed releases must instead provide reviewer, scope, reason, expiry, evidence, and target-enforcement fields in `security/permission_policy.json`.
Review Studio reads `reports/review_waivers.json` and links to `reports/review_waivers.md`.
references/skill-atlas-method.md
+7 -1
View File
@@ -17,6 +17,12 @@ Single-skill quality is not enough for a team library. A skill portfolio also ne
- Flag stale skills based on `updated_at` and `review_cadence`.
- Extract no-route opportunities from failure notes.
## Scope Policy
Atlas keeps a full catalog, but release gates should distinguish actionable library skills from examples and test fixtures.
Use `skill_atlas/policy.json` to mark path prefixes as non-actionable when they are intentionally retained as examples, evolution snapshots, embedded generated skills, or validator fixtures. Non-actionable items still appear in the full report, route matrix, stale list, and owner gap list, but Review Studio should use the actionable counts for release readiness.
## Reviewer Gate
Use Atlas before promoting a single skill into a shared library. If a route collision or missing owner appears, fix the portfolio boundary before adding more local complexity to one skill.
Use Atlas before promoting a single skill into a shared library. If an actionable route collision, missing owner, or stale governed skill appears, fix the portfolio boundary before adding more local complexity to one skill. Non-actionable issues should stay visible as evidence, not as release blockers.
references/telemetry-drift-method.md
+60
View File
@@ -0,0 +1,60 @@
# Telemetry And Drift Method
Telemetry turns real use into the next iteration queue. It must stay local-first and metadata-only by default.
## When To Use
Use the telemetry drift loop when a skill is production, library, governed, team-distributed, or repeatedly invoked by more than one workflow.
Do not collect raw prompts, model outputs, transcripts, notes, messages, or private files. If a reviewer needs examples, store anonymized fixtures separately and cite them as eval evidence, not telemetry.
## Event Contract
The local event stream is `reports/telemetry_events.jsonl`. It is intentionally narrow:
```json
{
"event": "skill_activation",
"skill": "example-skill",
"version": "2.0.0",
"activation_type": "implicit",
"outcome": "accepted",
"failure_type": "none",
"timestamp": "2026-06-13T10:00:00Z"
}
```
Allowed events: `skill_activation`, `skill_output`, `script_run`, `review_event`.
Allowed outcomes: `accepted`, `edited`, `rejected`, `missed`, `failed`, `reviewed`, `unknown`.
Allowed failure types: `wrong_trigger`, `under_trigger`, `bad_output`, `missing_resource`, `script_error`, `review_overdue`, `none`.
## Privacy Rule
The raw JSONL event log is local evidence and should not be distributed in skill packages. The distributable artifact is the aggregate report:
- `reports/adoption_drift_report.json`
- `reports/adoption_drift_report.md`
Package builders should exclude `reports/telemetry_events.jsonl`. The root repository also ignores this raw event stream so local usage evidence does not become ordinary source history by accident.
## Release Interpretation
- `no-data`: acceptable for a first scaffold, but a warning for governed release review.
- `low`: events exist and no drift failure signal is present.
- `medium`: at least one missed trigger, wrong trigger, bad output, script error, or overdue review signal exists.
- `high`: several drift signals are present; convert them into eval cases or governance actions before calling the skill release-ready.
## Iteration Loop
1. Capture metadata-only events locally.
2. Render `reports/adoption_drift_report.md`.
3. Convert missed triggers into trigger eval cases.
4. Convert bad outputs into Output Eval assertions and failure taxonomy entries.
5. Convert script errors into non-interactive smoke tests.
6. Feed review-overdue signals back into Skill Atlas and owner review.
## Review Studio Role
Review Studio should show the aggregate telemetry gate as an operating loop, not as raw logs. A blocker means the telemetry contract was violated. A warning means the evidence is absent or the drift signal needs a follow-up case.
references/trust-security-method.md
+53 -1
View File
@@ -16,11 +16,63 @@ Run the trust report when:
- obvious secret patterns
- script help surface and interactive prompts
- execution-level `--help` smoke checks
- dependency pinning
- runtime trust metadata
- network-capable scripts
- package integrity digest
- bounded host policy for network-capable scripts
- reviewer-approved permission policy for high-permission capabilities
- packaged-target runtime permission probes for adapter contracts and metadata fallback limits
- source-contract integrity digest
## Script Interface Rule
Every Python file under `scripts/` is reviewed as part of the package trust surface.
- CLI scripts should use `argparse` so reviewers and installers can run `python3 scripts/name.py --help` before execution.
- The trust report executes `python3 scripts/name.py --help` for CLI scripts with `argparse`, with a short timeout, and records pass/fail evidence.
- Import-only modules should declare `SCRIPT_INTERFACE = "internal-module"` near the top of the file.
- Internal modules should also declare `SCRIPT_INTERFACE_REASON` with a short explanation of which CLI or renderer imports them.
- The trust report keeps internal modules in the script inventory, but excludes them from CLI help-surface warnings.
- A Python file without an explicit internal-module declaration is treated as a CLI script by default.
- CLI scripts without `argparse` are not smoke-executed; they remain visible as help-surface warnings.
## Network Policy Rule
Network-capable scripts must be bounded by a machine-readable policy before team distribution.
- Put the policy in `security/network_policy.json`.
- Add one entry per network-capable script under `scripts`.
- Declare `allowed_hosts`, `allowed_path_prefixes`, purpose, timeout, auth mode, and custom-host behavior.
- Default to HTTPS-only and deny custom hosts unless a CLI flag or environment variable makes the override explicit.
- The trust report compares HTTPS URL literals in each script with `allowed_hosts`; missing or mismatched entries remain reviewer-visible warnings.
## Permission Approval Rule
High-permission capabilities must be approved before governed release.
- Put approvals in `security/permission_policy.json`.
- Cover each required capability detected by the trust report: `network`, `file_write`, `subprocess`, and `interactive` when present.
- Each approval must include `decision: approved`, `reviewer`, `scope`, `reason`, `expires_at`, `evidence`, and `target_enforcement`.
- Review Studio surfaces these checks as the `permission-gates` gate.
- Missing, invalid, or expired approvals block governed mode. They remain visible warnings in lighter modes.
## Runtime Permission Probe Rule
Permission approval validates reviewer intent. Runtime permission probes validate the generated target adapters after packaging.
- Run `python3 scripts/probe_runtime_permissions.py . --package-dir dist` after `cross_packager.py`.
- The probe writes `reports/runtime_permission_probes.json` and `reports/runtime_permission_probes.md`.
- A passing probe requires every target adapter to carry `permission_contract`, `target_permission_contract`, declared capabilities, a native-enforcement boolean, representation notes, and operator notes.
- If a target has no native enforcement, the probe must mark an explicit metadata fallback and keep residual risk reviewer-visible.
- Review Studio surfaces this as the `permission-runtime` gate.
## Release Rule
High-risk secrets or unrestricted remote inline execution block governed release. Warnings are reviewer-visible but do not block v0 unless the release owner decides the target environment requires stricter policy.
## Hash Scope
`package_sha256` is a stable source-contract digest, not a generated archive digest. It covers the skill entrypoint, metadata, scripts, references, evals, runtime, templates, security notes, Skill IR, and root control files. It deliberately excludes generated `reports/`, packaged `dist/` archives, and raw local telemetry so a report render or local adoption log cannot mutate the trust fingerprint.
Use the package verification or registry audit report for the distributable archive checksum.
registry/examples/yao-meta-skill-1.0.0.json
+43
View File
@@ -0,0 +1,43 @@
{
"schema_version": "2.0",
"name": "yao-meta-skill",
"version": "1.0.0",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.",
"targets": [
"openai",
"claude",
"generic"
],
"maturity": "governed",
"owner": "Yao Team",
"review_cadence": "quarterly",
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "0000000000000000000000000000000000000000000000000000000000000000"
},
"compatibility": {
"openai": "pass",
"claude": "pass",
"generic": "pass"
},
"source": {
"skill_dir": ".",
"skill_ir": "skill-ir/examples/yao-meta-skill.json",
"ir_schema_version": "2.0.0",
"canonical_metadata": "agents/interface.yaml"
},
"artifacts": {
"overview_html": "reports/skill-overview.html",
"review_studio_html": "reports/review-studio.html",
"trust_report": "reports/security_trust_report.md",
"conformance_matrix": "reports/conformance_matrix.md",
"package_metadata": "registry/packages/yao-meta-skill.json"
},
"distribution": {
"archive_verified": false,
"archive_sha256": "",
"package_verification": ""
},
"generated_at": "2026-06-12"
}
registry/index.json
+21
View File
@@ -0,0 +1,21 @@
{
"schema_version": "2.0",
"generated_at": "2026-06-13",
"package_count": 1,
"packages": [
{
"name": "yao-meta-skill",
"version": "1.1.0",
"maturity": "governed",
"owner": "Yao Team",
"targets": [
"openai",
"claude",
"generic",
"agent-skills-compatible"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe"
}
]
}
registry/index.schema.json
+30
View File
@@ -0,0 +1,30 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://yao.example/registry-index.schema.json",
"title": "Yao Skill Registry Index",
"type": "object",
"additionalProperties": false,
"required": ["schema_version", "generated_at", "package_count", "packages"],
"properties": {
"schema_version": {"const": "2.0"},
"generated_at": {"type": "string"},
"package_count": {"type": "integer", "minimum": 0},
"packages": {
"type": "array",
"items": {
"type": "object",
"additionalProperties": false,
"required": ["name", "version", "maturity", "owner", "targets", "package_metadata", "package_sha256"],
"properties": {
"name": {"type": "string", "minLength": 1},
"version": {"type": "string", "minLength": 1},
"maturity": {"type": "string", "minLength": 1},
"owner": {"type": "string", "minLength": 1},
"targets": {"type": "array", "items": {"type": "string"}},
"package_metadata": {"type": "string", "minLength": 1},
"package_sha256": {"type": "string", "minLength": 64, "maxLength": 64}
}
}
}
}
}
registry/package.schema.json
+53
View File
@@ -0,0 +1,53 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://yao.example/registry-package.schema.json",
"title": "Yao Skill Registry Package",
"type": "object",
"additionalProperties": false,
"required": [
"schema_version",
"name",
"version",
"description",
"targets",
"maturity",
"owner",
"review_cadence",
"trust_level",
"license",
"checksums",
"compatibility",
"source",
"artifacts",
"generated_at"
],
"properties": {
"schema_version": {"const": "2.0"},
"name": {"type": "string", "minLength": 1},
"version": {"type": "string", "minLength": 1},
"description": {"type": "string", "minLength": 1},
"targets": {"type": "array", "items": {"type": "string"}},
"maturity": {"type": "string", "minLength": 1},
"owner": {"type": "string", "minLength": 1},
"review_cadence": {"type": "string", "minLength": 1},
"trust_level": {"type": "string", "minLength": 1},
"license": {"type": "string", "minLength": 1},
"checksums": {
"type": "object",
"additionalProperties": false,
"required": ["package_sha256"],
"properties": {
"package_sha256": {"type": "string", "minLength": 64, "maxLength": 64},
"archive_sha256": {"type": "string", "minLength": 64, "maxLength": 64}
}
},
"compatibility": {
"type": "object",
"additionalProperties": {"enum": ["pass", "warn", "fail", "missing", "unknown"]}
},
"source": {"type": "object"},
"artifacts": {"type": "object"},
"distribution": {"type": "object"},
"generated_at": {"type": "string"}
}
}
registry/packages/yao-meta-skill.json
+56
View File
@@ -0,0 +1,56 @@
{
"schema_version": "2.0",
"name": "yao-meta-skill",
"version": "1.1.0",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"targets": [
"openai",
"claude",
"generic",
"agent-skills-compatible"
],
"maturity": "governed",
"owner": "Yao Team",
"review_cadence": "quarterly",
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe",
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06"
},
"compatibility": {
"openai": "pass",
"claude": "pass",
"agent-skills": "pass",
"vscode": "pass",
"generic": "pass",
"agent-skills-compatible": "pass"
},
"source": {
"skill_dir": ".",
"skill_ir": "skill-ir/examples/yao-meta-skill.json",
"ir_schema_version": "2.0.0",
"canonical_metadata": "agents/interface.yaml"
},
"artifacts": {
"overview_html": "reports/skill-overview.html",
"review_studio_html": "reports/review-studio.html",
"trust_report": "reports/security_trust_report.md",
"conformance_matrix": "reports/conformance_matrix.md",
"compiled_targets": "reports/compiled_targets.md",
"package_verification": "reports/package_verification.md",
"install_simulation": "reports/install_simulation.md",
"adoption_drift": "reports/adoption_drift_report.md",
"review_waivers": "reports/review_waivers.md",
"review_annotations": "reports/review_annotations.md",
"package_metadata": "registry/packages/yao-meta-skill.json"
},
"distribution": {
"archive_verified": true,
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
},
"generated_at": "2026-06-13"
}
reports/adoption_drift_report.json
+76
View File
@@ -0,0 +1,76 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13T09:57:51Z",
"skill_dir": ".",
"privacy_contract": {
"storage": "local-first",
"event_scope": "metadata-only",
"raw_content_allowed": false,
"raw_event_log_packaged": false,
"blocked_fields": [
"content",
"input",
"inputs",
"message",
"messages",
"note",
"output",
"outputs",
"prompt",
"raw",
"text",
"transcript"
]
},
"summary": {
"event_count": 1,
"activation_count": 1,
"accepted_count": 1,
"edited_count": 0,
"rejected_count": 0,
"missed_count": 0,
"failed_count": 0,
"adoption_rate": 100.0,
"missed_trigger_count": 0,
"wrong_trigger_count": 0,
"bad_output_count": 0,
"script_error_count": 0,
"missing_resource_count": 0,
"review_overdue_count": 0,
"risk_band": "low",
"event_types": {
"skill_activation": 1
},
"failure_types": {}
},
"adoption_by_skill": [
{
"skill": "yao-meta-skill",
"events": 1,
"accepted": 1,
"edited": 0,
"rejected": 0,
"missed": 0,
"adoption_rate": 100.0
}
],
"next_iteration_candidates": [],
"recent_events": [
{
"event": "skill_activation",
"skill": "yao-meta-skill",
"version": "1.1.0",
"activation_type": "explicit",
"outcome": "accepted",
"failure_type": "none",
"timestamp": "2026-06-13T10:00:00Z"
}
],
"failures": [],
"artifacts": {
"events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl",
"json": "reports/adoption_drift_report.json",
"markdown": "reports/adoption_drift_report.md"
}
}
reports/adoption_drift_report.md
+35
View File
@@ -0,0 +1,35 @@
# Adoption And Drift Report
Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, transcripts, and notes are not allowed in the event stream.
## Summary
- Events: `1`
- Activation events: `1`
- Adoption rate: `100.0`
- Missed trigger signals: `0`
- Bad output signals: `0`
- Script error signals: `0`
- Review overdue signals: `0`
- Risk band: `low`
## Privacy Contract
- Storage is local-first.
- Events are metadata-only.
- Raw user prompts, model outputs, transcripts, notes, and messages are blocked.
- Distributed packages should include this aggregate report, not raw `reports/telemetry_events.jsonl`.
## Adoption By Skill
| Skill | Events | Accepted | Edited | Rejected | Missed | Adoption Rate |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| `yao-meta-skill` | 1 | 1 | 0 | 0 | 0 | 100.0 |
## Next Iteration Candidates
- No telemetry-driven iteration candidate yet.
## Recent Metadata Events
- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` activation=`explicit` outcome=`accepted` failure=`none`
reports/artifact-design-profile.json
+13 -3
View File
@@ -1,6 +1,6 @@
{
"skill_name": "yao-meta-skill",
"design_system": "review studio",
"design_system": "metric editorial",
"primary_artifact": {
"key": "code_or_cli",
"label": "Code, CLI, or implementation guide",
@@ -36,12 +36,22 @@
{
"key": "review_viewer",
"label": "Review viewer",
"score": 1,
"score": 2,
"matched_keywords": [
"review"
"review",
"viewer"
],
"direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix."
},
{
"key": "dashboard",
"label": "Dashboard or metrics page",
"score": 1,
"matched_keywords": [
"table"
],
"direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation."
},
{
"key": "visual_capture",
"label": "Screenshot or visual evidence",
reports/artifact-design-profile.md
+8 -3
View File
@@ -1,7 +1,7 @@
# Artifact Design Profile
Skill: `yao-meta-skill`
Design system: `review studio`
Design system: `metric editorial`
## Primary Artifact Direction
@@ -22,10 +22,15 @@ Execution-focused technical artifact with environment assumptions, copyable comm
- Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
### Review viewer
- Matched keywords: review
- Score: `1`
- Matched keywords: review, viewer
- Score: `2`
- Direction: Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix.
### Dashboard or metrics page
- Matched keywords: table
- Score: `1`
- Direction: Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation.
### Screenshot or visual evidence
- Matched keywords: screenshot
- Score: `1`
reports/compiled_targets.json
+3130
View File
File diff suppressed because it is too large Load Diff
reports/compiled_targets.md
+63
View File
@@ -0,0 +1,63 @@
# Compiled Targets
- OK: `True`
- Targets: `4`
- Pass: `4`
- Warn: `0`
- Block: `0`
## Target Transforms
| Target | Status | Native Surface | Adapter Mode | Permissions | Degradation | Generated Files |
| --- | --- | --- | --- | --- | --- | --- |
| `openai` | `pass` | OpenAI-style interface metadata plus neutral Agent Skills source | `metadata-adapter` | `network, file_write, subprocess` | `metadata-adapter` | targets/openai/adapter.json, targets/openai/agents/openai.yaml |
| `claude` | `pass` | Claude-compatible neutral source folder with adapter notes | `neutral-source-plus-adapter` | `network, file_write, subprocess` | `neutral-source-plus-adapter` | targets/claude/adapter.json, targets/claude/README.md |
| `generic` | `pass` | Agent Skills compatible neutral package | `agent-skills-compatible` | `network, file_write, subprocess` | `neutral-source` | targets/generic/adapter.json |
| `agent-skills-compatible` | `pass` | Agent Skills standard source tree | `neutral-agent-skills-source` | `network, file_write, subprocess` | `neutral-source` | SKILL.md, agents/interface.yaml |
## Native Behavior Contracts
### openai
- Native surface: OpenAI-style interface metadata plus neutral Agent Skills source
- Activation: Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.
- Resources: Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.
- Scripts: Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.
- Permission enforcement: `metadata-only`; native enforcement `False`
- Review artifacts: targets/openai/agents/openai.yaml, targets/openai/adapter.json, reports/review-studio.html
### claude
- Native surface: Claude-compatible neutral source folder with adapter notes
- Activation: Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.
- Resources: Preserve the source tree directly; write target notes in targets/claude/README.md.
- Scripts: Scripts remain local package resources and must be reviewed through trust and permission reports before use.
- Permission enforcement: `metadata-fallback`; native enforcement `False`
- Review artifacts: targets/claude/README.md, targets/claude/adapter.json, reports/review-studio.html
### generic
- Native surface: Agent Skills compatible neutral package
- Activation: Use SKILL.md name and description; consumers decide automatic or manual activation.
- Resources: Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.
- Scripts: Expose script and permission metadata for downstream clients or installers to enforce.
- Permission enforcement: `consumer-enforced-or-metadata-only`; native enforcement `False`
- Review artifacts: targets/generic/adapter.json, reports/review-studio.html
### agent-skills-compatible
- Native surface: Agent Skills standard source tree
- Activation: Use SKILL.md frontmatter name and description for progressive disclosure.
- Resources: Keep optional directories as relative resources next to SKILL.md.
- Scripts: Scripts remain local optional resources and should advertise --help when executable.
- Permission enforcement: `consumer-enforced-or-metadata-only`; native enforcement `False`
- Review artifacts: SKILL.md, agents/interface.yaml, reports/review-studio.html
## Failures
- None
## Warnings
- None
reports/conformance_matrix.json
+12 -12
View File
@@ -32,6 +32,7 @@
"Skill IR description matches frontmatter",
"references resource resolves: references/artifact-design-doctrine.md",
"references resource resolves: references/authoring-discipline.md",
"references resource resolves: references/distribution-registry-method.md",
"references resource resolves: references/eval-playbook.md",
"references resource resolves: references/gate-selection.md",
"references resource resolves: references/governance.md",
@@ -40,8 +41,7 @@
"references resource resolves: references/iteration-philosophy.md",
"references resource resolves: references/non-skill-decision-tree.md",
"references resource resolves: references/operating-modes.md",
"references resource resolves: references/output-eval-method.md",
"references resource resolves: references/output-quality-risk.md"
"references resource resolves: references/output-eval-method.md"
],
"failures": [],
"warnings": []
@@ -76,6 +76,7 @@
"Skill IR description matches frontmatter",
"references resource resolves: references/artifact-design-doctrine.md",
"references resource resolves: references/authoring-discipline.md",
"references resource resolves: references/distribution-registry-method.md",
"references resource resolves: references/eval-playbook.md",
"references resource resolves: references/gate-selection.md",
"references resource resolves: references/governance.md",
@@ -84,8 +85,7 @@
"references resource resolves: references/iteration-philosophy.md",
"references resource resolves: references/non-skill-decision-tree.md",
"references resource resolves: references/operating-modes.md",
"references resource resolves: references/output-eval-method.md",
"references resource resolves: references/output-quality-risk.md"
"references resource resolves: references/output-eval-method.md"
],
"failures": [],
"warnings": []
@@ -120,6 +120,7 @@
"Skill IR description matches frontmatter",
"references resource resolves: references/artifact-design-doctrine.md",
"references resource resolves: references/authoring-discipline.md",
"references resource resolves: references/distribution-registry-method.md",
"references resource resolves: references/eval-playbook.md",
"references resource resolves: references/gate-selection.md",
"references resource resolves: references/governance.md",
@@ -128,12 +129,11 @@
"references resource resolves: references/iteration-philosophy.md",
"references resource resolves: references/non-skill-decision-tree.md",
"references resource resolves: references/operating-modes.md",
"references resource resolves: references/output-eval-method.md",
"references resource resolves: references/output-quality-risk.md"
"references resource resolves: references/output-eval-method.md"
],
"failures": [],
"warnings": [
"agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0."
"agent-skills uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0."
]
},
{
@@ -166,6 +166,7 @@
"Skill IR description matches frontmatter",
"references resource resolves: references/artifact-design-doctrine.md",
"references resource resolves: references/authoring-discipline.md",
"references resource resolves: references/distribution-registry-method.md",
"references resource resolves: references/eval-playbook.md",
"references resource resolves: references/gate-selection.md",
"references resource resolves: references/governance.md",
@@ -174,12 +175,11 @@
"references resource resolves: references/iteration-philosophy.md",
"references resource resolves: references/non-skill-decision-tree.md",
"references resource resolves: references/operating-modes.md",
"references resource resolves: references/output-eval-method.md",
"references resource resolves: references/output-quality-risk.md"
"references resource resolves: references/output-eval-method.md"
],
"failures": [],
"warnings": [
"vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0."
"vscode uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0."
]
},
{
@@ -212,6 +212,7 @@
"Skill IR description matches frontmatter",
"references resource resolves: references/artifact-design-doctrine.md",
"references resource resolves: references/authoring-discipline.md",
"references resource resolves: references/distribution-registry-method.md",
"references resource resolves: references/eval-playbook.md",
"references resource resolves: references/gate-selection.md",
"references resource resolves: references/governance.md",
@@ -220,8 +221,7 @@
"references resource resolves: references/iteration-philosophy.md",
"references resource resolves: references/non-skill-decision-tree.md",
"references resource resolves: references/operating-modes.md",
"references resource resolves: references/output-eval-method.md",
"references resource resolves: references/output-quality-risk.md"
"references resource resolves: references/output-eval-method.md"
],
"failures": [],
"warnings": []
reports/conformance_matrix.md
+2 -2
View File
@@ -9,8 +9,8 @@
| --- | --- | --- | --- |
| openai | pass | None | None |
| claude | pass | None | None |
| agent-skills | pass | None | agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0. |
| vscode | pass | None | vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0. |
| agent-skills | pass | None | agent-skills uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0. |
| vscode | pass | None | vscode uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0. |
| generic | pass | None | None |
## Reviewer Notes
reports/context_budget.json
+6 -6
View File
@@ -5,13 +5,13 @@
"stats": {
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 734,
"other_text_tokens": 606468,
"estimated_initial_load_tokens": 910,
"estimated_total_text_tokens": 607202,
"relevant_file_count": 241,
"skill_body_tokens": 811,
"other_text_tokens": 864534,
"estimated_initial_load_tokens": 987,
"estimated_total_text_tokens": 865345,
"relevant_file_count": 316,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 142.9
"quality_density": 131.7
}
}
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |
| root | `.` | `production` | 1000 | 910 | 734 | 142.9 | - | ok |
| root | `.` | `production` | 1000 | 987 | 811 | 131.7 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok |
reports/context_budget_summary.json
+3 -3
View File
@@ -6,9 +6,9 @@
"path": ".",
"budget_tier": "production",
"budget_limit": 1000,
"initial_tokens": 910,
"skill_body_tokens": 734,
"quality_density": 142.9,
"initial_tokens": 987,
"skill_body_tokens": 811,
"quality_density": 131.7,
"unused_resource_dirs": [],
"ok": true,
"warnings": []
reports/install_simulation.json
+121
View File
@@ -0,0 +1,121 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13",
"skill_dir": ".",
"package_dir": "dist",
"install_root": "dist/install-simulation/simulate-yao-meta-skill",
"installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill",
"summary": {
"archive_present": true,
"archive_entry_count": 483,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
"interface_loaded": true,
"adapter_count": 3,
"install_root_is_temp": false,
"failure_count": 0,
"warning_count": 0
},
"checks": [
{
"id": "archive-present",
"status": "pass",
"detail": "Package archive exists: dist/yao-meta-skill.zip"
},
{
"id": "archive-safe-paths",
"status": "pass",
"detail": "Archive has no absolute or parent-traversal entries"
},
{
"id": "single-top-level",
"status": "pass",
"detail": "Archive top-level directory is yao-meta-skill"
},
{
"id": "entrypoint-load",
"status": "pass",
"detail": "Installed SKILL.md frontmatter is readable"
},
{
"id": "entrypoint-name",
"status": "pass",
"detail": "Installed SKILL.md name matches package directory"
},
{
"id": "entrypoint-description",
"status": "pass",
"detail": "Installed SKILL.md description is present"
},
{
"id": "manifest-load",
"status": "pass",
"detail": "Installed manifest.json is readable"
},
{
"id": "manifest-name",
"status": "pass",
"detail": "Installed manifest name matches package manifest"
},
{
"id": "manifest-version",
"status": "pass",
"detail": "Installed manifest version matches package manifest"
},
{
"id": "interface-load",
"status": "pass",
"detail": "Installed agents/interface.yaml is readable"
},
{
"id": "overview-report",
"status": "pass",
"detail": "Installed overview report is present"
},
{
"id": "review-studio-report",
"status": "pass",
"detail": "Installed Review Studio report is present"
},
{
"id": "adapter-claude",
"status": "pass",
"detail": "claude adapter is readable after package install simulation"
},
{
"id": "adapter-claude-name",
"status": "pass",
"detail": "claude adapter name matches package manifest"
},
{
"id": "adapter-generic",
"status": "pass",
"detail": "generic adapter is readable after package install simulation"
},
{
"id": "adapter-generic-name",
"status": "pass",
"detail": "generic adapter name matches package manifest"
},
{
"id": "adapter-openai",
"status": "pass",
"detail": "openai adapter is readable after package install simulation"
},
{
"id": "adapter-openai-name",
"status": "pass",
"detail": "openai adapter name matches package manifest"
}
],
"failures": [],
"warnings": [],
"artifacts": {
"archive": "dist/yao-meta-skill.zip",
"package_manifest": "dist/manifest.json",
"json": "reports/install_simulation.json",
"markdown": "reports/install_simulation.md"
}
}
reports/install_simulation.md
+42
View File
@@ -0,0 +1,42 @@
# Install Simulation
- OK: `True`
- Package directory: `dist`
- Archive extracted: `True`
- Entrypoint loaded: `True`
- Manifest loaded: `True`
- Interface loaded: `True`
- Adapters readable: `3`
- Failures: `0`
- Warnings: `0`
## Checks
| Check | Status | Detail |
| --- | --- | --- |
| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip |
| `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries |
| `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill |
| `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable |
| `entrypoint-name` | `pass` | Installed SKILL.md name matches package directory |
| `entrypoint-description` | `pass` | Installed SKILL.md description is present |
| `manifest-load` | `pass` | Installed manifest.json is readable |
| `manifest-name` | `pass` | Installed manifest name matches package manifest |
| `manifest-version` | `pass` | Installed manifest version matches package manifest |
| `interface-load` | `pass` | Installed agents/interface.yaml is readable |
| `overview-report` | `pass` | Installed overview report is present |
| `review-studio-report` | `pass` | Installed Review Studio report is present |
| `adapter-claude` | `pass` | claude adapter is readable after package install simulation |
| `adapter-claude-name` | `pass` | claude adapter name matches package manifest |
| `adapter-generic` | `pass` | generic adapter is readable after package install simulation |
| `adapter-generic-name` | `pass` | generic adapter name matches package manifest |
| `adapter-openai` | `pass` | openai adapter is readable after package install simulation |
| `adapter-openai-name` | `pass` | openai adapter name matches package manifest |
## Failures
- None
## Warnings
- None
reports/intent-confidence.json
+46 -72
View File
@@ -1,79 +1,53 @@
{
"score": 30,
"band": "low",
"gate_passed": false,
"score": 100,
"band": "high",
"gate_passed": true,
"strengths": [
"The recurring job is concrete enough to anchor the package."
"The recurring job is concrete enough to anchor the package.",
"Real input shape is explicit.",
"The hand-back output is concrete.",
"Boundary exclusions are already explicit.",
"Operational constraints are visible."
],
"gaps": [
{
"key": "real_inputs",
"label": "Real inputs are missing",
"reason": "Without real inputs, it is hard to choose assets, scripts, or examples.",
"severity": "high"
},
{
"key": "primary_output",
"label": "Primary output is missing",
"reason": "The package does not yet know what it must hand back.",
"severity": "high"
},
{
"key": "exclusions",
"label": "Near-neighbor exclusions are missing",
"reason": "The route may blur into nearby requests without an exclusion list.",
"severity": "high"
},
{
"key": "constraints",
"label": "Constraints are missing",
"reason": "The package does not yet know which tradeoffs matter most.",
"severity": "high"
},
{
"key": "standards",
"label": "Quality bar is implied, not explicit",
"reason": "The first evaluation target is still underspecified.",
"severity": "medium"
}
],
"follow_up_questions": [
{
"slot": "real_inputs",
"question": "What material will people actually hand to this skill in practice?",
"why": "Real input shape decides whether references, scripts, or examples are needed.",
"list": true,
"label": "Real inputs are missing",
"severity": "high"
},
{
"slot": "primary_output",
"question": "What finished hand-back should this skill return so the next person can keep moving?",
"why": "The output is the anchor for package design and review.",
"list": false,
"label": "Primary output is missing",
"severity": "high"
},
{
"slot": "exclusions",
"question": "What nearby requests should this skill clearly leave out so the boundary stays clean?",
"why": "Exclusions are the fastest route to better trigger quality.",
"list": true,
"label": "Near-neighbor exclusions are missing",
"severity": "high"
}
],
"anchor_sentence": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.",
"gaps": [],
"follow_up_questions": [],
"anchor_sentence": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.",
"recommended_action": "Intent is clear enough to package the first routeable version.",
"context": {
"job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"real_inputs": [],
"primary_output": "",
"job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"real_inputs": [
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"primary_output": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"exclusions": [],
"constraints": [],
"standards": [],
"correction": "",
"user_references": []
"exclusions": [
"one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill",
"general code review or debugging unless the user is packaging that workflow as a skill",
"raw private material that was not intentionally supplied as skill evidence",
"platform-specific plugin creation when the request is not about an agent skill package"
],
"constraints": [
"keep SKILL.md lean and route primarily through frontmatter description",
"put durable guidance in references, executable logic in scripts, and evidence in reports",
"default to the lightest reliable mode before adding governance weight",
"preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets",
"avoid raw prompt, output, transcript, or private content in telemetry"
],
"standards": [
"trigger boundaries must be tested with should-trigger and should-not-trigger cases",
"production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence",
"governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation",
"generated reports should be bilingual or reviewer-friendly when they are user-facing",
"each new asset must earn its place by reducing ambiguity, risk, or repeated work"
],
"correction": "The 2.0 direction reframes this package from a Meta Skill factory into a Skill OS with Skill IR, compiler contracts, output eval, runtime conformance, trust, registry, Review Studio, Skill Atlas, and telemetry loops.",
"user_references": [
"Yao Meta Skill 2.0 upgrade plan",
"Kami-style white report layout requirements for generated skill reports",
"local Waza engineering habits for review, design, diagnosis, and implementation discipline"
]
}
}
reports/intent-confidence.md
+11 -16
View File
@@ -1,31 +1,26 @@
# Intent Confidence
- Confidence score: `30/100`
- Confidence band: `low`
- Gate passed: `False`
- Recommended action: Pause before deep authoring and close the highest-leverage gaps first.
- Confidence score: `100/100`
- Confidence band: `high`
- Gate passed: `True`
- Recommended action: Intent is clear enough to package the first routeable version.
## Current Reading
Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.
## Strong Signals
- The recurring job is concrete enough to anchor the package.
- Real input shape is explicit.
- The hand-back output is concrete.
- Boundary exclusions are already explicit.
- Operational constraints are visible.
## Gaps To Close
- **Real inputs are missing** (`high`): Without real inputs, it is hard to choose assets, scripts, or examples.
- **Primary output is missing** (`high`): The package does not yet know what it must hand back.
- **Near-neighbor exclusions are missing** (`high`): The route may blur into nearby requests without an exclusion list.
- **Constraints are missing** (`high`): The package does not yet know which tradeoffs matter most.
- **Quality bar is implied, not explicit** (`medium`): The first evaluation target is still underspecified.
- No major intent gaps detected.
## Follow-Up Questions
- **What material will people actually hand to this skill in practice?**
- Why: Real input shape decides whether references, scripts, or examples are needed.
- **What finished hand-back should this skill return so the next person can keep moving?**
- Why: The output is the anchor for package design and review.
- **What nearby requests should this skill clearly leave out so the boundary stays clean?**
- Why: Exclusions are the fastest route to better trigger quality.
- No extra follow-up questions required before the first package.
reports/intent-context.json
+34 -8
View File
@@ -1,11 +1,37 @@
{
"job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"real_inputs": [],
"primary_output": "",
"job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"real_inputs": [
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"primary_output": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"exclusions": [],
"constraints": [],
"standards": [],
"correction": "",
"user_references": []
"exclusions": [
"one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill",
"general code review or debugging unless the user is packaging that workflow as a skill",
"raw private material that was not intentionally supplied as skill evidence",
"platform-specific plugin creation when the request is not about an agent skill package"
],
"constraints": [
"keep SKILL.md lean and route primarily through frontmatter description",
"put durable guidance in references, executable logic in scripts, and evidence in reports",
"default to the lightest reliable mode before adding governance weight",
"preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets",
"avoid raw prompt, output, transcript, or private content in telemetry"
],
"standards": [
"trigger boundaries must be tested with should-trigger and should-not-trigger cases",
"production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence",
"governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation",
"generated reports should be bilingual or reviewer-friendly when they are user-facing",
"each new asset must earn its place by reducing ambiguity, risk, or repeated work"
],
"correction": "The 2.0 direction reframes this package from a Meta Skill factory into a Skill OS with Skill IR, compiler contracts, output eval, runtime conformance, trust, registry, Review Studio, Skill Atlas, and telemetry loops.",
"user_references": [
"Yao Meta Skill 2.0 upgrade plan",
"Kami-style white report layout requirements for generated skill reports",
"local Waza engineering habits for review, design, diagnosis, and implementation discipline"
]
}
reports/output_blind_answer_key.json
+56
View File
@@ -0,0 +1,56 @@
{
"schema_version": "1.0",
"seed": "yao-output-eval-blind-v1",
"summary": {
"pair_count": 5,
"with_skill_expected_count": 5,
"baseline_expected_count": 0
},
"answers": [
{
"case_id": "skill-package-contract",
"variant_a_role": "with_skill",
"variant_b_role": "baseline",
"expected_winner_role": "with_skill",
"expected_winner_variant": "A",
"score_winner_role": "with_skill",
"delta": 100.0
},
{
"case_id": "output-eval-expectation",
"variant_a_role": "with_skill",
"variant_b_role": "baseline",
"expected_winner_role": "with_skill",
"expected_winner_variant": "A",
"score_winner_role": "with_skill",
"delta": 100.0
},
{
"case_id": "ir-before-packaging",
"variant_a_role": "baseline",
"variant_b_role": "with_skill",
"expected_winner_role": "with_skill",
"expected_winner_variant": "B",
"score_winner_role": "with_skill",
"delta": 100.0
},
{
"case_id": "near-neighbor-boundary",
"variant_a_role": "baseline",
"variant_b_role": "with_skill",
"expected_winner_role": "with_skill",
"expected_winner_variant": "B",
"score_winner_role": "with_skill",
"delta": 100.0
},
{
"case_id": "file-backed-governed-package",
"variant_a_role": "baseline",
"variant_b_role": "with_skill",
"expected_winner_role": "with_skill",
"expected_winner_variant": "B",
"score_winner_role": "with_skill",
"delta": 100.0
}
]
}
reports/output_blind_review_pack.json
+204
View File
@@ -0,0 +1,204 @@
{
"schema_version": "1.0",
"seed": "yao-output-eval-blind-v1",
"summary": {
"pair_count": 5,
"answer_key_separate": true,
"with_skill_hidden_count": 5
},
"pairs": [
{
"case_id": "skill-package-contract",
"prompt": "Turn this repeated workflow into a reusable team skill package.",
"input_files": [],
"metadata": {},
"review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.",
"rubric": [
{
"id": "has-entrypoint",
"description": "Output names the routeable Skill entrypoint.",
"weight": 1.0
},
{
"id": "has-interface",
"description": "Output includes neutral interface metadata.",
"weight": 1.0
},
{
"id": "has-report",
"description": "Output points reviewer to generated overview report.",
"weight": 1.0
},
{
"id": "has-resource-boundary",
"description": "Output preserves entrypoint/resource separation.",
"weight": 1.0
}
],
"variant_a": {
"blind_id": "skill-package-contract:A",
"output": "Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/."
},
"variant_b": {
"blind_id": "skill-package-contract:B",
"output": "I can write a prompt for that workflow and include a short checklist."
}
},
{
"case_id": "output-eval-expectation",
"prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.",
"input_files": [],
"metadata": {},
"review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.",
"rubric": [
{
"id": "has-baseline-comparison",
"description": "Output explicitly compares with-skill and baseline outputs.",
"weight": 1.0
},
{
"id": "has-assertion-grading",
"description": "Output includes objective assertion grading.",
"weight": 1.0
},
{
"id": "has-scorecard",
"description": "Output produces a scorecard report path.",
"weight": 1.0
},
{
"id": "has-failure-taxonomy",
"description": "Output records failure taxonomy.",
"weight": 1.0
}
],
"variant_a": {
"blind_id": "output-eval-expectation:A",
"output": "Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md."
},
"variant_b": {
"blind_id": "output-eval-expectation:B",
"output": "Add more examples and run the trigger eval again."
}
},
{
"case_id": "ir-before-packaging",
"prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.",
"input_files": [],
"metadata": {},
"review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.",
"rubric": [
{
"id": "has-ir-first",
"description": "Output requires Skill IR before packaging.",
"weight": 1.0
},
{
"id": "has-core-ir-fields",
"description": "Output lists core IR contract fields.",
"weight": 1.0
},
{
"id": "has-targets",
"description": "Output names requested runtime targets.",
"weight": 1.0
},
{
"id": "has-semantic-preservation",
"description": "Output says targets must preserve the capability contract.",
"weight": 1.0
}
],
"variant_a": {
"blind_id": "ir-before-packaging:A",
"output": "Export folders for each platform and adjust files as needed."
},
"variant_b": {
"blind_id": "ir-before-packaging:B",
"output": "Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract."
}
},
{
"case_id": "near-neighbor-boundary",
"prompt": "I only need a one-off summary of these notes, not a reusable process.",
"input_files": [],
"metadata": {
"case_type": "near_neighbor",
"tier": "production"
},
"review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.",
"rubric": [
{
"id": "declines-skill",
"description": "Output refuses unnecessary skill creation for one-off work.",
"weight": 1.0
},
{
"id": "names-near-neighbor",
"description": "Output labels the request as near-neighbor instead of owned work.",
"weight": 1.0
},
{
"id": "requires-reuse-signal",
"description": "Output asks for repeat-use evidence before packaging.",
"weight": 1.0
}
],
"variant_a": {
"blind_id": "near-neighbor-boundary:A",
"output": "Create a SKILL.md and a reusable workflow anyway so future notes can use it."
},
"variant_b": {
"blind_id": "near-neighbor-boundary:B",
"output": "Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract."
}
},
{
"case_id": "file-backed-governed-package",
"prompt": "Turn the attached release brief source into a governed skill package.",
"input_files": [
"fixtures/release-brief-source.md"
],
"metadata": {
"case_type": "boundary",
"tier": "governed"
},
"review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.",
"rubric": [
{
"id": "uses-file-backed-evidence",
"description": "Output names file-backed source evidence.",
"weight": 1.0
},
{
"id": "has-governance",
"description": "Output preserves governed ownership metadata.",
"weight": 1.0
},
{
"id": "has-output-contract",
"description": "Output preserves output and rollback boundaries.",
"weight": 1.0
},
{
"id": "has-trust-and-scorecard",
"description": "Output requires trust and output scorecard artifacts.",
"weight": 1.0
},
{
"id": "does-not-invent-evidence",
"description": "Output forbids invented launch evidence.",
"weight": 1.0
}
],
"variant_a": {
"blind_id": "file-backed-governed-package:A",
"output": "Draft a release announcement with the changelog, support notes, and owner name."
},
"variant_b": {
"blind_id": "file-backed-governed-package:B",
"output": "Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them."
}
}
]
}
reports/output_blind_review_pack.md
+97
View File
@@ -0,0 +1,97 @@
# Output Blind A/B Review Pack
This packet hides whether each variant came from the baseline or the skill-guided output. Use the separate answer key only after review.
- Pairs: `5`
- Seed: `yao-output-eval-blind-v1`
- Answer key separate: `True`
## Case: skill-package-contract
Prompt: Turn this repeated workflow into a reusable team skill package.
Rubric:
- `has-entrypoint` (1.0): Output names the routeable Skill entrypoint.
- `has-interface` (1.0): Output includes neutral interface metadata.
- `has-report` (1.0): Output points reviewer to generated overview report.
- `has-resource-boundary` (1.0): Output preserves entrypoint/resource separation.
### Variant A
Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/.
### Variant B
I can write a prompt for that workflow and include a short checklist.
## Case: output-eval-expectation
Prompt: Upgrade this production skill so we know whether the generated output is better than baseline.
Rubric:
- `has-baseline-comparison` (1.0): Output explicitly compares with-skill and baseline outputs.
- `has-assertion-grading` (1.0): Output includes objective assertion grading.
- `has-scorecard` (1.0): Output produces a scorecard report path.
- `has-failure-taxonomy` (1.0): Output records failure taxonomy.
### Variant A
Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md.
### Variant B
Add more examples and run the trigger eval again.
## Case: ir-before-packaging
Prompt: Package this skill for OpenAI, Claude, Agent Skills, and generic targets.
Rubric:
- `has-ir-first` (1.0): Output requires Skill IR before packaging.
- `has-core-ir-fields` (1.0): Output lists core IR contract fields.
- `has-targets` (1.0): Output names requested runtime targets.
- `has-semantic-preservation` (1.0): Output says targets must preserve the capability contract.
### Variant A
Export folders for each platform and adjust files as needed.
### Variant B
Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract.
## Case: near-neighbor-boundary
Prompt: I only need a one-off summary of these notes, not a reusable process.
Rubric:
- `declines-skill` (1.0): Output refuses unnecessary skill creation for one-off work.
- `names-near-neighbor` (1.0): Output labels the request as near-neighbor instead of owned work.
- `requires-reuse-signal` (1.0): Output asks for repeat-use evidence before packaging.
### Variant A
Create a SKILL.md and a reusable workflow anyway so future notes can use it.
### Variant B
Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract.
## Case: file-backed-governed-package
Prompt: Turn the attached release brief source into a governed skill package.
Rubric:
- `uses-file-backed-evidence` (1.0): Output names file-backed source evidence.
- `has-governance` (1.0): Output preserves governed ownership metadata.
- `has-output-contract` (1.0): Output preserves output and rollback boundaries.
- `has-trust-and-scorecard` (1.0): Output requires trust and output scorecard artifacts.
- `does-not-invent-evidence` (1.0): Output forbids invented launch evidence.
### Variant A
Draft a release announcement with the changelog, support notes, and owner name.
### Variant B
Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them.
reports/output_execution_runs.json
+291
View File
@@ -0,0 +1,291 @@
{
"schema_version": "1.0",
"ok": true,
"cases": "evals/output/cases.jsonl",
"runner": {
"mode": "command",
"command": [
"python3",
"scripts/local_output_eval_runner.py"
],
"timeout_seconds": 30.0
},
"summary": {
"case_count": 5,
"variant_run_count": 10,
"command_executed_count": 10,
"model_executed_count": 0,
"recorded_fixture_count": 0,
"timing_observed_count": 10,
"token_observed_count": 0,
"token_estimated_count": 10,
"baseline_pass_rate": 0.0,
"with_skill_pass_rate": 100.0,
"delta": 100.0,
"regression_count": 0,
"failure_count": 0,
"gate_pass": true
},
"runs": [
{
"case_id": "skill-package-contract",
"variant": "baseline",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.22,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 16,
"output_tokens": 17,
"total_tokens": 33,
"estimated": true
},
"score": 0.0,
"passed_count": 0,
"failed_count": 4,
"failed_assertions": [
"has-entrypoint",
"has-interface",
"has-report",
"has-resource-boundary"
],
"output_sha256": "ece26a6edbbbea86a93faaf9f0eadf88efbae12f1c3ad81da9376cf2a889d101",
"failure": ""
},
{
"case_id": "skill-package-contract",
"variant": "with_skill",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.12,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 16,
"output_tokens": 57,
"total_tokens": 73,
"estimated": true
},
"score": 100.0,
"passed_count": 4,
"failed_count": 0,
"failed_assertions": [],
"output_sha256": "8bee0131e17fdbe87e48457130ac6aeaa55ea30c919a45e77109852609f33eba",
"failure": ""
},
{
"case_id": "output-eval-expectation",
"variant": "baseline",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.12,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 24,
"output_tokens": 12,
"total_tokens": 36,
"estimated": true
},
"score": 0.0,
"passed_count": 0,
"failed_count": 4,
"failed_assertions": [
"has-baseline-comparison",
"has-assertion-grading",
"has-scorecard",
"has-failure-taxonomy"
],
"output_sha256": "98fa332fb493b5dc14656a9d65a6c0a9c25ff61a14b71b647aad89cbdbd3ab35",
"failure": ""
},
{
"case_id": "output-eval-expectation",
"variant": "with_skill",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.03,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 24,
"output_tokens": 56,
"total_tokens": 80,
"estimated": true
},
"score": 100.0,
"passed_count": 4,
"failed_count": 0,
"failed_assertions": [],
"output_sha256": "8efdbfefa6a36603682b7063b1753f9f939b3810c082d29686e297906d9f51c8",
"failure": ""
},
{
"case_id": "ir-before-packaging",
"variant": "baseline",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.11,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 18,
"output_tokens": 15,
"total_tokens": 33,
"estimated": true
},
"score": 0.0,
"passed_count": 0,
"failed_count": 4,
"failed_assertions": [
"has-ir-first",
"has-core-ir-fields",
"has-targets",
"has-semantic-preservation"
],
"output_sha256": "07a4aa9ee9e48341d4b84d5cb9b6414901aa2c8841340d67447e187240c6c4c2",
"failure": ""
},
{
"case_id": "ir-before-packaging",
"variant": "with_skill",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.68,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 18,
"output_tokens": 62,
"total_tokens": 80,
"estimated": true
},
"score": 100.0,
"passed_count": 4,
"failed_count": 0,
"failed_assertions": [],
"output_sha256": "707fde419b3d61e921561b1c4a8523e1f7b53b1acd5758a47ce21d9ed711fde0",
"failure": ""
},
{
"case_id": "near-neighbor-boundary",
"variant": "baseline",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.07,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 17,
"output_tokens": 19,
"total_tokens": 36,
"estimated": true
},
"score": 0.0,
"passed_count": 0,
"failed_count": 3,
"failed_assertions": [
"declines-skill",
"names-near-neighbor",
"requires-reuse-signal"
],
"output_sha256": "648ebaa6e09486d05f33fa6639dc973b029d115158588fd192efbe15ccaaeac4",
"failure": ""
},
{
"case_id": "near-neighbor-boundary",
"variant": "with_skill",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.57,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 17,
"output_tokens": 48,
"total_tokens": 65,
"estimated": true
},
"score": 100.0,
"passed_count": 3,
"failed_count": 0,
"failed_assertions": [],
"output_sha256": "17a3a5286ad573dea86e51f82c04b596babaecb92cde8879f09e7c519205e2e1",
"failure": ""
},
{
"case_id": "file-backed-governed-package",
"variant": "baseline",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.2,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 17,
"output_tokens": 20,
"total_tokens": 37,
"estimated": true
},
"score": 0.0,
"passed_count": 0,
"failed_count": 5,
"failed_assertions": [
"uses-file-backed-evidence",
"has-governance",
"has-output-contract",
"has-trust-and-scorecard",
"does-not-invent-evidence"
],
"output_sha256": "6e5827219459dee34143e4f11663415db80c3111291f75fb6c2115c7bda4cf30",
"failure": ""
},
{
"case_id": "file-backed-governed-package",
"variant": "with_skill",
"status": "pass",
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.94,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
"input_tokens": 17,
"output_tokens": 81,
"total_tokens": 98,
"estimated": true
},
"score": 100.0,
"passed_count": 5,
"failed_count": 0,
"failed_assertions": [],
"output_sha256": "bcfdf036666d406abbc945fc42b93eb74d7074118893e540253c7d8715baad73",
"failure": ""
}
],
"failures": [],
"artifacts": {
"json": "reports/output_execution_runs.json",
"markdown": "reports/output_execution_runs.md"
}
}
reports/output_execution_runs.md
+41
View File
@@ -0,0 +1,41 @@
# Output Execution Runs
This report records how output-eval variants were produced and whether timing or token evidence is observed or estimated.
- Cases: `5`
- Variant runs: `10`
- Command executed: `10`
- Model executed: `0`
- Recorded fixtures: `0`
- Timing observed: `10`
- Token observed: `0`
- Token estimated: `10`
- Delta: `100.0`
- Gate pass: `True`
No model-executed runs are recorded yet.
Use `--runner-command` with a provider-backed runner to replace recorded fixtures with real model output evidence.
Command runner evidence is present. This proves the eval harness executed an external command, but it is not provider-backed model evidence unless the runner reports model metadata.
## Runs
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 27.22 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.12 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.12 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 30.03 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 27.11 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.68 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.07 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.57 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 29.2 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.94 | 98 | 100.0 | pass |
## Next Fixes
- Keep recorded fixtures as reproducible baselines, but do not describe them as model-executed evidence.
- Add provider-backed command runners for holdout cases when release confidence depends on real generation behavior.
- Compare timing, token cost, and assertion deltas before promoting a skill to governed reuse.
reports/output_quality_scorecard.json
+11 -2
View File
@@ -11,7 +11,8 @@
"delta": 100.0,
"regression_count": 0,
"gate_pass": true,
"failure_taxonomy": []
"failure_taxonomy": [],
"blind_pair_count": 5
},
"results": [
{
@@ -800,8 +801,16 @@
}
],
"failures": [],
"blind_review": {
"pack": "reports/output_blind_review_pack.json",
"answer_key": "reports/output_blind_answer_key.json",
"pair_count": 5
},
"artifacts": {
"json": "reports/output_quality_scorecard.json",
"markdown": "reports/output_quality_scorecard.md"
"markdown": "reports/output_quality_scorecard.md",
"blind_review_pack_json": "reports/output_blind_review_pack.json",
"blind_review_pack_md": "reports/output_blind_review_pack.md",
"blind_answer_key_json": "reports/output_blind_answer_key.json"
}
}
reports/output_quality_scorecard.md
+4
View File
@@ -7,8 +7,12 @@ This v0 scorecard compares static without-skill and with-skill outputs using ass
- With-skill pass rate: `100.0`
- Delta: `100.0`
- Regressions: `0`
- Blind A/B pairs: `5`
- Gate pass: `True`
Blind review artifacts are generated separately so reviewers can inspect A/B outputs without seeing the answer key.
Run output review adjudication after reviewer decisions are recorded; pending cases should stay pending rather than being counted as human agreement.
## Case Results
| Case | Baseline | With Skill | Delta | Winner | Failed With-Skill Assertions |
reports/output_review_adjudication.json
+73
View File
@@ -0,0 +1,73 @@
{
"schema_version": "1.0",
"ok": true,
"summary": {
"pair_count": 5,
"judgment_count": 0,
"pending_count": 5,
"agreement_count": 0,
"disagreement_count": 0,
"invalid_decision_count": 0,
"agreement_rate": null,
"needs_review": true,
"failure_count": 0
},
"reviewer": "",
"reviewed_at": "",
"artifacts": {
"blind_pack": "reports/output_blind_review_pack.json",
"answer_key": "reports/output_blind_answer_key.json",
"decisions": "reports/output_review_decisions.json",
"json": "reports/output_review_adjudication.json",
"markdown": "reports/output_review_adjudication.md"
},
"template_written": false,
"pairs": [
{
"case_id": "skill-package-contract",
"status": "pending",
"expected_winner_variant": "A",
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
"prompt": "Turn this repeated workflow into a reusable team skill package."
},
{
"case_id": "output-eval-expectation",
"status": "pending",
"expected_winner_variant": "A",
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
"prompt": "Upgrade this production skill so we know whether the generated output is better than baseline."
},
{
"case_id": "ir-before-packaging",
"status": "pending",
"expected_winner_variant": "B",
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
"prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets."
},
{
"case_id": "near-neighbor-boundary",
"status": "pending",
"expected_winner_variant": "B",
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
"prompt": "I only need a one-off summary of these notes, not a reusable process."
},
{
"case_id": "file-backed-governed-package",
"status": "pending",
"expected_winner_variant": "B",
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
"prompt": "Turn the attached release brief source into a governed skill package."
}
],
"failures": []
}
reports/output_review_adjudication.md
+29
View File
@@ -0,0 +1,29 @@
# Output Review Adjudication
This report adjudicates reviewer choices from the blind A/B output review pack against the separate answer key.
- Pairs: `5`
- Judgments: `0`
- Pending: `5`
- Agreement rate: `n/a`
- Invalid decisions: `0`
No reviewer decisions recorded yet.
Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.
## Case Adjudication
| Case | Reviewer | Expected | Status | Confidence | Reason |
| --- | --- | --- | --- | ---: | --- |
| skill-package-contract | pending | A | pending | | |
| output-eval-expectation | pending | A | pending | | |
| ir-before-packaging | pending | B | pending | | |
| near-neighbor-boundary | pending | B | pending | | |
| file-backed-governed-package | pending | B | pending | | |
## Next Fixes
- Keep the blind review pack separate from the answer key until decisions are recorded.
- Treat disagreement cases as prompts for rubric tuning or output improvement.
- Add model-executed holdout runs after this human adjudication harness is stable.
reports/package_verification.json
+516
View File
@@ -0,0 +1,516 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13",
"skill_dir": ".",
"package_dir": "dist",
"summary": {
"target_count": 3,
"adapter_count": 3,
"archive_present": true,
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06",
"archive_entry_count": 483,
"failure_count": 0,
"warning_count": 0
},
"checks": [
{
"id": "package-manifest",
"status": "pass",
"detail": "Package manifest exists: dist/manifest.json"
},
{
"id": "openai-adapter",
"status": "pass",
"detail": "Adapter exists for target: openai"
},
{
"id": "openai-field-name",
"status": "pass",
"detail": "openai adapter includes field: name"
},
{
"id": "openai-field-description",
"status": "pass",
"detail": "openai adapter includes field: description"
},
{
"id": "openai-field-version",
"status": "pass",
"detail": "openai adapter includes field: version"
},
{
"id": "openai-field-display_name",
"status": "pass",
"detail": "openai adapter includes field: display_name"
},
{
"id": "openai-field-short_description",
"status": "pass",
"detail": "openai adapter includes field: short_description"
},
{
"id": "openai-field-default_prompt",
"status": "pass",
"detail": "openai adapter includes field: default_prompt"
},
{
"id": "openai-field-job_to_be_done",
"status": "pass",
"detail": "openai adapter includes field: job_to_be_done"
},
{
"id": "openai-field-ir_source",
"status": "pass",
"detail": "openai adapter includes field: ir_source"
},
{
"id": "openai-field-ir_schema_version",
"status": "pass",
"detail": "openai adapter includes field: ir_schema_version"
},
{
"id": "openai-field-semantic_contract",
"status": "pass",
"detail": "openai adapter includes field: semantic_contract"
},
{
"id": "openai-field-semantic_parity",
"status": "pass",
"detail": "openai adapter includes field: semantic_parity"
},
{
"id": "openai-field-compiler",
"status": "pass",
"detail": "openai adapter includes field: compiler"
},
{
"id": "openai-field-compiled_contract",
"status": "pass",
"detail": "openai adapter includes field: compiled_contract"
},
{
"id": "openai-field-permission_contract",
"status": "pass",
"detail": "openai adapter includes field: permission_contract"
},
{
"id": "openai-field-target_permission_contract",
"status": "pass",
"detail": "openai adapter includes field: target_permission_contract"
},
{
"id": "openai-field-target_native_contract",
"status": "pass",
"detail": "openai adapter includes field: target_native_contract"
},
{
"id": "openai-field-target_transform",
"status": "pass",
"detail": "openai adapter includes field: target_transform"
},
{
"id": "openai-field-canonical_metadata",
"status": "pass",
"detail": "openai adapter includes field: canonical_metadata"
},
{
"id": "openai-field-canonical_format",
"status": "pass",
"detail": "openai adapter includes field: canonical_format"
},
{
"id": "openai-field-activation_mode",
"status": "pass",
"detail": "openai adapter includes field: activation_mode"
},
{
"id": "openai-field-execution_context",
"status": "pass",
"detail": "openai adapter includes field: execution_context"
},
{
"id": "openai-field-shell",
"status": "pass",
"detail": "openai adapter includes field: shell"
},
{
"id": "openai-field-trust_level",
"status": "pass",
"detail": "openai adapter includes field: trust_level"
},
{
"id": "openai-field-remote_inline_execution",
"status": "pass",
"detail": "openai adapter includes field: remote_inline_execution"
},
{
"id": "openai-field-degradation_strategy",
"status": "pass",
"detail": "openai adapter includes field: degradation_strategy"
},
{
"id": "openai-field-portability_profile",
"status": "pass",
"detail": "openai adapter includes field: portability_profile"
},
{
"id": "claude-adapter",
"status": "pass",
"detail": "Adapter exists for target: claude"
},
{
"id": "claude-field-name",
"status": "pass",
"detail": "claude adapter includes field: name"
},
{
"id": "claude-field-description",
"status": "pass",
"detail": "claude adapter includes field: description"
},
{
"id": "claude-field-version",
"status": "pass",
"detail": "claude adapter includes field: version"
},
{
"id": "claude-field-display_name",
"status": "pass",
"detail": "claude adapter includes field: display_name"
},
{
"id": "claude-field-short_description",
"status": "pass",
"detail": "claude adapter includes field: short_description"
},
{
"id": "claude-field-default_prompt",
"status": "pass",
"detail": "claude adapter includes field: default_prompt"
},
{
"id": "claude-field-job_to_be_done",
"status": "pass",
"detail": "claude adapter includes field: job_to_be_done"
},
{
"id": "claude-field-ir_source",
"status": "pass",
"detail": "claude adapter includes field: ir_source"
},
{
"id": "claude-field-ir_schema_version",
"status": "pass",
"detail": "claude adapter includes field: ir_schema_version"
},
{
"id": "claude-field-semantic_contract",
"status": "pass",
"detail": "claude adapter includes field: semantic_contract"
},
{
"id": "claude-field-semantic_parity",
"status": "pass",
"detail": "claude adapter includes field: semantic_parity"
},
{
"id": "claude-field-compiler",
"status": "pass",
"detail": "claude adapter includes field: compiler"
},
{
"id": "claude-field-compiled_contract",
"status": "pass",
"detail": "claude adapter includes field: compiled_contract"
},
{
"id": "claude-field-permission_contract",
"status": "pass",
"detail": "claude adapter includes field: permission_contract"
},
{
"id": "claude-field-target_permission_contract",
"status": "pass",
"detail": "claude adapter includes field: target_permission_contract"
},
{
"id": "claude-field-target_native_contract",
"status": "pass",
"detail": "claude adapter includes field: target_native_contract"
},
{
"id": "claude-field-target_transform",
"status": "pass",
"detail": "claude adapter includes field: target_transform"
},
{
"id": "claude-field-canonical_metadata",
"status": "pass",
"detail": "claude adapter includes field: canonical_metadata"
},
{
"id": "claude-field-canonical_format",
"status": "pass",
"detail": "claude adapter includes field: canonical_format"
},
{
"id": "claude-field-activation_mode",
"status": "pass",
"detail": "claude adapter includes field: activation_mode"
},
{
"id": "claude-field-execution_context",
"status": "pass",
"detail": "claude adapter includes field: execution_context"
},
{
"id": "claude-field-shell",
"status": "pass",
"detail": "claude adapter includes field: shell"
},
{
"id": "claude-field-trust_level",
"status": "pass",
"detail": "claude adapter includes field: trust_level"
},
{
"id": "claude-field-remote_inline_execution",
"status": "pass",
"detail": "claude adapter includes field: remote_inline_execution"
},
{
"id": "claude-field-degradation_strategy",
"status": "pass",
"detail": "claude adapter includes field: degradation_strategy"
},
{
"id": "claude-field-portability_profile",
"status": "pass",
"detail": "claude adapter includes field: portability_profile"
},
{
"id": "generic-adapter",
"status": "pass",
"detail": "Adapter exists for target: generic"
},
{
"id": "generic-field-name",
"status": "pass",
"detail": "generic adapter includes field: name"
},
{
"id": "generic-field-description",
"status": "pass",
"detail": "generic adapter includes field: description"
},
{
"id": "generic-field-version",
"status": "pass",
"detail": "generic adapter includes field: version"
},
{
"id": "generic-field-display_name",
"status": "pass",
"detail": "generic adapter includes field: display_name"
},
{
"id": "generic-field-short_description",
"status": "pass",
"detail": "generic adapter includes field: short_description"
},
{
"id": "generic-field-default_prompt",
"status": "pass",
"detail": "generic adapter includes field: default_prompt"
},
{
"id": "generic-field-job_to_be_done",
"status": "pass",
"detail": "generic adapter includes field: job_to_be_done"
},
{
"id": "generic-field-ir_source",
"status": "pass",
"detail": "generic adapter includes field: ir_source"
},
{
"id": "generic-field-ir_schema_version",
"status": "pass",
"detail": "generic adapter includes field: ir_schema_version"
},
{
"id": "generic-field-semantic_contract",
"status": "pass",
"detail": "generic adapter includes field: semantic_contract"
},
{
"id": "generic-field-semantic_parity",
"status": "pass",
"detail": "generic adapter includes field: semantic_parity"
},
{
"id": "generic-field-compiler",
"status": "pass",
"detail": "generic adapter includes field: compiler"
},
{
"id": "generic-field-compiled_contract",
"status": "pass",
"detail": "generic adapter includes field: compiled_contract"
},
{
"id": "generic-field-permission_contract",
"status": "pass",
"detail": "generic adapter includes field: permission_contract"
},
{
"id": "generic-field-target_permission_contract",
"status": "pass",
"detail": "generic adapter includes field: target_permission_contract"
},
{
"id": "generic-field-target_native_contract",
"status": "pass",
"detail": "generic adapter includes field: target_native_contract"
},
{
"id": "generic-field-target_transform",
"status": "pass",
"detail": "generic adapter includes field: target_transform"
},
{
"id": "generic-field-canonical_metadata",
"status": "pass",
"detail": "generic adapter includes field: canonical_metadata"
},
{
"id": "generic-field-canonical_format",
"status": "pass",
"detail": "generic adapter includes field: canonical_format"
},
{
"id": "generic-field-activation_mode",
"status": "pass",
"detail": "generic adapter includes field: activation_mode"
},
{
"id": "generic-field-execution_context",
"status": "pass",
"detail": "generic adapter includes field: execution_context"
},
{
"id": "generic-field-shell",
"status": "pass",
"detail": "generic adapter includes field: shell"
},
{
"id": "generic-field-trust_level",
"status": "pass",
"detail": "generic adapter includes field: trust_level"
},
{
"id": "generic-field-remote_inline_execution",
"status": "pass",
"detail": "generic adapter includes field: remote_inline_execution"
},
{
"id": "generic-field-degradation_strategy",
"status": "pass",
"detail": "generic adapter includes field: degradation_strategy"
},
{
"id": "generic-field-portability_profile",
"status": "pass",
"detail": "generic adapter includes field: portability_profile"
},
{
"id": "openai-file-targets/openai/adapter.json",
"status": "pass",
"detail": "Package contains targets/openai/adapter.json"
},
{
"id": "openai-file-targets/openai/agents/openai.yaml",
"status": "pass",
"detail": "Package contains targets/openai/agents/openai.yaml"
},
{
"id": "claude-file-targets/claude/adapter.json",
"status": "pass",
"detail": "Package contains targets/claude/adapter.json"
},
{
"id": "claude-file-targets/claude/README.md",
"status": "pass",
"detail": "Package contains targets/claude/README.md"
},
{
"id": "generic-file-targets/generic/adapter.json",
"status": "pass",
"detail": "Package contains targets/generic/adapter.json"
},
{
"id": "archive-safe-paths",
"status": "pass",
"detail": "Archive has no absolute or parent-traversal entries"
},
{
"id": "archive-entry-yao-meta-skill/SKILL.md",
"status": "pass",
"detail": "Archive contains yao-meta-skill/SKILL.md"
},
{
"id": "archive-entry-yao-meta-skill/manifest.json",
"status": "pass",
"detail": "Archive contains yao-meta-skill/manifest.json"
},
{
"id": "archive-entry-yao-meta-skill/agents/interface.yaml",
"status": "pass",
"detail": "Archive contains yao-meta-skill/agents/interface.yaml"
},
{
"id": "archive-excludes-generated",
"status": "pass",
"detail": "Archive excludes generated dist/ and tests/tmp* contents"
},
{
"id": "registry-ok",
"status": "pass",
"detail": "Registry audit is OK"
},
{
"id": "registry-name-match",
"status": "pass",
"detail": "Registry package name matches package manifest"
},
{
"id": "registry-version-match",
"status": "pass",
"detail": "Registry package version matches package manifest"
},
{
"id": "registry-compat-openai",
"status": "pass",
"detail": "Registry compatibility is reviewable for target: openai"
},
{
"id": "registry-compat-claude",
"status": "pass",
"detail": "Registry compatibility is reviewable for target: claude"
},
{
"id": "registry-compat-generic",
"status": "pass",
"detail": "Registry compatibility is reviewable for target: generic"
}
],
"failures": [],
"warnings": [],
"artifacts": {
"manifest": "dist/manifest.json",
"archive": "dist/yao-meta-skill.zip",
"json": "reports/package_verification.json",
"markdown": "reports/package_verification.md"
}
}
reports/package_verification.md
+120
View File
@@ -0,0 +1,120 @@
# Package Verification
- OK: `True`
- Package directory: `dist`
- Targets: `3 / 3` adapters present
- Archive present: `True`
- Archive SHA256: `4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06`
- Failures: `0`
- Warnings: `0`
## Checks
| Check | Status | Detail |
| --- | --- | --- |
| `package-manifest` | `pass` | Package manifest exists: dist/manifest.json |
| `openai-adapter` | `pass` | Adapter exists for target: openai |
| `openai-field-name` | `pass` | openai adapter includes field: name |
| `openai-field-description` | `pass` | openai adapter includes field: description |
| `openai-field-version` | `pass` | openai adapter includes field: version |
| `openai-field-display_name` | `pass` | openai adapter includes field: display_name |
| `openai-field-short_description` | `pass` | openai adapter includes field: short_description |
| `openai-field-default_prompt` | `pass` | openai adapter includes field: default_prompt |
| `openai-field-job_to_be_done` | `pass` | openai adapter includes field: job_to_be_done |
| `openai-field-ir_source` | `pass` | openai adapter includes field: ir_source |
| `openai-field-ir_schema_version` | `pass` | openai adapter includes field: ir_schema_version |
| `openai-field-semantic_contract` | `pass` | openai adapter includes field: semantic_contract |
| `openai-field-semantic_parity` | `pass` | openai adapter includes field: semantic_parity |
| `openai-field-compiler` | `pass` | openai adapter includes field: compiler |
| `openai-field-compiled_contract` | `pass` | openai adapter includes field: compiled_contract |
| `openai-field-permission_contract` | `pass` | openai adapter includes field: permission_contract |
| `openai-field-target_permission_contract` | `pass` | openai adapter includes field: target_permission_contract |
| `openai-field-target_native_contract` | `pass` | openai adapter includes field: target_native_contract |
| `openai-field-target_transform` | `pass` | openai adapter includes field: target_transform |
| `openai-field-canonical_metadata` | `pass` | openai adapter includes field: canonical_metadata |
| `openai-field-canonical_format` | `pass` | openai adapter includes field: canonical_format |
| `openai-field-activation_mode` | `pass` | openai adapter includes field: activation_mode |
| `openai-field-execution_context` | `pass` | openai adapter includes field: execution_context |
| `openai-field-shell` | `pass` | openai adapter includes field: shell |
| `openai-field-trust_level` | `pass` | openai adapter includes field: trust_level |
| `openai-field-remote_inline_execution` | `pass` | openai adapter includes field: remote_inline_execution |
| `openai-field-degradation_strategy` | `pass` | openai adapter includes field: degradation_strategy |
| `openai-field-portability_profile` | `pass` | openai adapter includes field: portability_profile |
| `claude-adapter` | `pass` | Adapter exists for target: claude |
| `claude-field-name` | `pass` | claude adapter includes field: name |
| `claude-field-description` | `pass` | claude adapter includes field: description |
| `claude-field-version` | `pass` | claude adapter includes field: version |
| `claude-field-display_name` | `pass` | claude adapter includes field: display_name |
| `claude-field-short_description` | `pass` | claude adapter includes field: short_description |
| `claude-field-default_prompt` | `pass` | claude adapter includes field: default_prompt |
| `claude-field-job_to_be_done` | `pass` | claude adapter includes field: job_to_be_done |
| `claude-field-ir_source` | `pass` | claude adapter includes field: ir_source |
| `claude-field-ir_schema_version` | `pass` | claude adapter includes field: ir_schema_version |
| `claude-field-semantic_contract` | `pass` | claude adapter includes field: semantic_contract |
| `claude-field-semantic_parity` | `pass` | claude adapter includes field: semantic_parity |
| `claude-field-compiler` | `pass` | claude adapter includes field: compiler |
| `claude-field-compiled_contract` | `pass` | claude adapter includes field: compiled_contract |
| `claude-field-permission_contract` | `pass` | claude adapter includes field: permission_contract |
| `claude-field-target_permission_contract` | `pass` | claude adapter includes field: target_permission_contract |
| `claude-field-target_native_contract` | `pass` | claude adapter includes field: target_native_contract |
| `claude-field-target_transform` | `pass` | claude adapter includes field: target_transform |
| `claude-field-canonical_metadata` | `pass` | claude adapter includes field: canonical_metadata |
| `claude-field-canonical_format` | `pass` | claude adapter includes field: canonical_format |
| `claude-field-activation_mode` | `pass` | claude adapter includes field: activation_mode |
| `claude-field-execution_context` | `pass` | claude adapter includes field: execution_context |
| `claude-field-shell` | `pass` | claude adapter includes field: shell |
| `claude-field-trust_level` | `pass` | claude adapter includes field: trust_level |
| `claude-field-remote_inline_execution` | `pass` | claude adapter includes field: remote_inline_execution |
| `claude-field-degradation_strategy` | `pass` | claude adapter includes field: degradation_strategy |
| `claude-field-portability_profile` | `pass` | claude adapter includes field: portability_profile |
| `generic-adapter` | `pass` | Adapter exists for target: generic |
| `generic-field-name` | `pass` | generic adapter includes field: name |
| `generic-field-description` | `pass` | generic adapter includes field: description |
| `generic-field-version` | `pass` | generic adapter includes field: version |
| `generic-field-display_name` | `pass` | generic adapter includes field: display_name |
| `generic-field-short_description` | `pass` | generic adapter includes field: short_description |
| `generic-field-default_prompt` | `pass` | generic adapter includes field: default_prompt |
| `generic-field-job_to_be_done` | `pass` | generic adapter includes field: job_to_be_done |
| `generic-field-ir_source` | `pass` | generic adapter includes field: ir_source |
| `generic-field-ir_schema_version` | `pass` | generic adapter includes field: ir_schema_version |
| `generic-field-semantic_contract` | `pass` | generic adapter includes field: semantic_contract |
| `generic-field-semantic_parity` | `pass` | generic adapter includes field: semantic_parity |
| `generic-field-compiler` | `pass` | generic adapter includes field: compiler |
| `generic-field-compiled_contract` | `pass` | generic adapter includes field: compiled_contract |
| `generic-field-permission_contract` | `pass` | generic adapter includes field: permission_contract |
| `generic-field-target_permission_contract` | `pass` | generic adapter includes field: target_permission_contract |
| `generic-field-target_native_contract` | `pass` | generic adapter includes field: target_native_contract |
| `generic-field-target_transform` | `pass` | generic adapter includes field: target_transform |
| `generic-field-canonical_metadata` | `pass` | generic adapter includes field: canonical_metadata |
| `generic-field-canonical_format` | `pass` | generic adapter includes field: canonical_format |
| `generic-field-activation_mode` | `pass` | generic adapter includes field: activation_mode |
| `generic-field-execution_context` | `pass` | generic adapter includes field: execution_context |
| `generic-field-shell` | `pass` | generic adapter includes field: shell |
| `generic-field-trust_level` | `pass` | generic adapter includes field: trust_level |
| `generic-field-remote_inline_execution` | `pass` | generic adapter includes field: remote_inline_execution |
| `generic-field-degradation_strategy` | `pass` | generic adapter includes field: degradation_strategy |
| `generic-field-portability_profile` | `pass` | generic adapter includes field: portability_profile |
| `openai-file-targets/openai/adapter.json` | `pass` | Package contains targets/openai/adapter.json |
| `openai-file-targets/openai/agents/openai.yaml` | `pass` | Package contains targets/openai/agents/openai.yaml |
| `claude-file-targets/claude/adapter.json` | `pass` | Package contains targets/claude/adapter.json |
| `claude-file-targets/claude/README.md` | `pass` | Package contains targets/claude/README.md |
| `generic-file-targets/generic/adapter.json` | `pass` | Package contains targets/generic/adapter.json |
| `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries |
| `archive-entry-yao-meta-skill/SKILL.md` | `pass` | Archive contains yao-meta-skill/SKILL.md |
| `archive-entry-yao-meta-skill/manifest.json` | `pass` | Archive contains yao-meta-skill/manifest.json |
| `archive-entry-yao-meta-skill/agents/interface.yaml` | `pass` | Archive contains yao-meta-skill/agents/interface.yaml |
| `archive-excludes-generated` | `pass` | Archive excludes generated dist/ and tests/tmp* contents |
| `registry-ok` | `pass` | Registry audit is OK |
| `registry-name-match` | `pass` | Registry package name matches package manifest |
| `registry-version-match` | `pass` | Registry package version matches package manifest |
| `registry-compat-openai` | `pass` | Registry compatibility is reviewable for target: openai |
| `registry-compat-claude` | `pass` | Registry compatibility is reviewable for target: claude |
| `registry-compat-generic` | `pass` | Registry compatibility is reviewable for target: generic |
## Failures
- None
## Warnings
- None
reports/prompt-quality-profile.json
+38 -24
View File
@@ -2,14 +2,26 @@
"skill_name": "yao-meta-skill",
"relevance": "prompt-heavy",
"primary_task_family": {
"key": "dialogue_interaction",
"label": "Dialogue interaction",
"key": "execution_operation",
"label": "Execution operation",
"matched_keywords": [
"dialogue",
"conversation"
"workflow",
"runbook"
]
},
"task_families": [
{
"key": "execution_operation",
"label": "Execution operation",
"score": 2,
"matched_keywords": [
"workflow",
"runbook"
],
"role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.",
"task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.",
"format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant."
},
{
"key": "dialogue_interaction",
"label": "Dialogue interaction",
@@ -23,15 +35,15 @@
"format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions."
},
{
"key": "execution_operation",
"label": "Execution operation",
"key": "creative_generation",
"label": "Creative generation",
"score": 1,
"matched_keywords": [
"workflow"
"content"
],
"role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.",
"task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.",
"format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant."
"role_guidance": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.",
"task_guidance": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.",
"format_guidance": "Return options with rationale, selection criteria, and refinement paths."
},
{
"key": "prompt_engineering",
@@ -46,27 +58,27 @@
}
],
"complexity": {
"band": "complex",
"score": 5,
"reason": "multiple inputs, constraints, or task families require tradeoff handling"
"band": "expert",
"score": 27,
"reason": "multiple task families plus governance, evaluation, or expert-level constraints"
},
"need_model": {
"explicit_need": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"explicit_need": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"implicit_need": "The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.",
"scenario": "not yet explicit",
"scenario": "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions, an existing skill directory that needs refactor, evaluation, packaging, or governance hardening, target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution, benchmark references, local constraints, desired maturity tier, and review standards",
"user_level": "infer from examples and standards; ask only if it changes output depth",
"success_standard": "usable output with clear validation cues"
"success_standard": "trigger boundaries must be tested with should-trigger and should-not-trigger cases, production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence, governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation, generated reports should be bilingual or reviewer-friendly when they are user-facing, each new asset must earn its place by reducing ambiguity, risk, or repeated work"
},
"rtf_to_skill": {
"role": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.",
"task": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.",
"format": "Return concise prompts, decision points, and reviewer-visible assumptions."
"role": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.",
"task": "Convert the job into ordered steps with validation checks and stop conditions.",
"format": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant."
},
"quality_matrix": [
{
"key": "completeness",
"label": "Completeness",
"score": 80,
"score": 100,
"matched_signals": [
"output",
"constraint",
@@ -77,8 +89,10 @@
{
"key": "clarity",
"label": "Clarity",
"score": 80,
"matched_signals": [],
"score": 85,
"matched_signals": [
"clear"
],
"repair": "Replace broad verbs with observable actions and define what done means."
},
{
@@ -104,12 +118,12 @@
{
"key": "specificity",
"label": "Specificity",
"score": 70,
"score": 80,
"matched_signals": [],
"repair": "Anchor wording in the user's audience, domain nouns, and target outcome."
}
],
"overall_quality_score": 82.0,
"overall_quality_score": 89.0,
"self_repair_checks": [
"Check explicit need, implicit need, scenario, user level, and success standard before deepening.",
"Map Role, Task, and Format into skill behavior, not decorative prompt labels.",
reports/prompt-quality-profile.md
+28 -21
View File
@@ -2,41 +2,41 @@
Skill: `yao-meta-skill`
Relevance: `prompt-heavy`
Overall quality score: `82.0/100`
Overall quality score: `89.0/100`
## Primary Task Family
**Dialogue interaction**
- Matched keywords: dialogue, conversation
**Execution operation**
- Matched keywords: workflow, runbook
## Complexity
- Band: `complex`
- Score: `5`
- Reason: multiple inputs, constraints, or task families require tradeoff handling
- Band: `expert`
- Score: `27`
- Reason: multiple task families plus governance, evaluation, or expert-level constraints
## Need Model
- Explicit Need: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
- Explicit Need: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.
- Implicit Need: The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.
- Scenario: not yet explicit
- Scenario: rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions, an existing skill directory that needs refactor, evaluation, packaging, or governance hardening, target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution, benchmark references, local constraints, desired maturity tier, and review standards
- User Level: infer from examples and standards; ask only if it changes output depth
- Success Standard: usable output with clear validation cues
- Success Standard: trigger boundaries must be tested with should-trigger and should-not-trigger cases, production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence, governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation, generated reports should be bilingual or reviewer-friendly when they are user-facing, each new asset must earn its place by reducing ambiguity, risk, or repeated work
## RTF To Skill Mapping
- Role: Use a conversational role that asks only high-leverage questions and remembers the user's goal.
- Task: Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.
- Format: Return concise prompts, decision points, and reviewer-visible assumptions.
- Role: Use an operator role with explicit boundaries, inputs, outputs, and failure handling.
- Task: Convert the job into ordered steps with validation checks and stop conditions.
- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant.
## Quality Matrix
### Completeness — 80/100
### Completeness — 100/100
- Matched signals: output, constraint, standard
- Repair: Name missing inputs, outputs, constraints, or success standards before deepening the package.
### Clarity — 80/100
- Matched signals: none
### Clarity — 85/100
- Matched signals: clear
- Repair: Replace broad verbs with observable actions and define what done means.
### Consistency — 90/100
@@ -47,12 +47,19 @@ Overall quality score: `82.0/100`
- Matched signals: use, workflow
- Repair: Add runnable steps, examples, or verification cues instead of abstract advice.
### Specificity — 70/100
### Specificity — 80/100
- Matched signals: none
- Repair: Anchor wording in the user's audience, domain nouns, and target outcome.
## Matched Task Families
### Execution operation
- Score: `2`
- Keywords: workflow, runbook
- Role: Use an operator role with explicit boundaries, inputs, outputs, and failure handling.
- Task: Convert the job into ordered steps with validation checks and stop conditions.
- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant.
### Dialogue interaction
- Score: `2`
- Keywords: dialogue, conversation
@@ -60,12 +67,12 @@ Overall quality score: `82.0/100`
- Task: Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.
- Format: Return concise prompts, decision points, and reviewer-visible assumptions.
### Execution operation
### Creative generation
- Score: `1`
- Keywords: workflow
- Role: Use an operator role with explicit boundaries, inputs, outputs, and failure handling.
- Task: Convert the job into ordered steps with validation checks and stop conditions.
- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant.
- Keywords: content
- Role: Use a taste-aware creator role with clear audience, tone, and originality boundaries.
- Task: Generate variants, explain selection logic, and preserve the user's distinctive constraints.
- Format: Return options with rationale, selection criteria, and refinement paths.
### Prompt engineering
- Score: `1`
reports/reference-synthesis.json
+80 -73
View File
@@ -2,9 +2,9 @@
"skill_name": "yao-meta-skill",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"intent_confidence": {
"score": 30,
"band": "low",
"gate_passed": false
"score": 100,
"band": "high",
"gate_passed": true
},
"github_benchmarks": [
{
@@ -38,49 +38,51 @@
"source_tracks": [
{
"source_type": "official",
"name": "Official workflow product ergonomics",
"name": "Official skill anatomy and context discipline",
"evidence_mode": "curated-pattern-track",
"matched_keywords": [
"workflow"
"metadata",
"references"
],
"borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.",
"avoid": "Do not mimic product polish that adds UI bulk without improving clarity.",
"why_relevant": "This track matches: workflow."
"borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"avoid": "Do not let packaging or platform concerns swallow the core job boundary.",
"why_relevant": "This track matches: metadata, references."
},
{
"source_type": "research",
"name": "Hypothesis-test-learn loop",
"name": "Human-in-the-loop verification",
"evidence_mode": "curated-pattern-track",
"matched_keywords": [
"general fit"
"review",
"govern"
],
"borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.",
"avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.",
"why_relevant": "This track matches: general fit."
"borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.",
"avoid": "Do not force every skill through heavyweight review when the risk is low.",
"why_relevant": "This track matches: review, govern."
},
{
"source_type": "principles",
"name": "Boundary-first design",
"evidence_mode": "curated-pattern-track",
"matched_keywords": [
"general fit"
"route"
],
"borrow": "Borrow the discipline of defining what the skill should not own before growing the package.",
"avoid": "Do not expand execution assets until route boundaries stay clean.",
"why_relevant": "This track matches: general fit."
"why_relevant": "This track matches: route."
}
],
"synthesis": {
"borrow_now": [
"Borrow a first-time operator flow that explains itself before it asks for more structure.",
"Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.",
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed.",
"Borrow the discipline of defining what the skill should not own before growing the package.",
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure."
],
"avoid_now": [
"Do not mimic product polish that adds UI bulk without improving clarity.",
"Do not create experimental overhead that exceeds the skill's real risk tier.",
"Do not let packaging or platform concerns swallow the core job boundary.",
"Do not force every skill through heavyweight review when the risk is low.",
"Do not expand execution assets until route boundaries stay clean.",
"Do not import process overhead that only exists for that project's scale.",
"Do not copy repo-specific commands or environment assumptions verbatim."
@@ -96,10 +98,50 @@
"source_count": 6,
"accepted": [
{
"name": "Official workflow product ergonomics",
"name": "Official skill anatomy and context discipline",
"source_type": "official",
"borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.",
"avoid": "Do not mimic product polish that adds UI bulk without improving clarity.",
"borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"avoid": "Do not let packaging or platform concerns swallow the core job boundary.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
},
{
"name": "Human-in-the-loop verification",
"source_type": "research",
"borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.",
"avoid": "Do not force every skill through heavyweight review when the risk is low.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
},
{
"name": "Boundary-first design",
"source_type": "principles",
"borrow": "Borrow the discipline of defining what the skill should not own before growing the package.",
"avoid": "Do not expand execution assets until route boundaries stay clean.",
"gates": {
"recurrence": true,
"generativity": true,
@@ -157,48 +199,6 @@
}
],
"deferred": [
{
"name": "Hypothesis-test-learn loop",
"source_type": "research",
"borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.",
"avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": false,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"boundary"
],
"missing": [
"distinctiveness"
],
"score": 3
},
{
"name": "Boundary-first design",
"source_type": "principles",
"borrow": "Borrow the discipline of defining what the skill should not own before growing the package.",
"avoid": "Do not expand execution assets until route boundaries stay clean.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": false,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"boundary"
],
"missing": [
"distinctiveness"
],
"score": 3
},
{
"name": "multica-ai/andrej-karpathy-skills",
"source_type": "github",
@@ -221,27 +221,34 @@
"score": 3
}
],
"summary": "3 accepted, 3 deferred using threshold 4/4."
"summary": "5 accepted, 1 deferred using threshold 4/4."
},
"conflicts": [],
"conflicts": [
{
"key": "lightweight_vs_governance",
"summary": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.",
"user_preference": "lightweight or speed-first",
"benchmark_pressure": "governance or evaluation-heavy patterns"
}
],
"recommendation": {
"summary": "Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.",
"summary": "Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary.",
"borrow_now": [
"Borrow a first-time operator flow that explains itself before it asks for more structure.",
"Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed."
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed."
],
"avoid_for_now": [
"Do not mimic product polish that adds UI bulk without improving clarity.",
"Do not create experimental overhead that exceeds the skill's real risk tier."
"Do not let packaging or platform concerns swallow the core job boundary.",
"Do not force every skill through heavyweight review when the risk is low."
],
"why": "Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package.",
"why": "There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.",
"user_decision_required": true
},
"visibility": {
"mode": "explicit",
"user_decision_required": true,
"reasons": [
"intent_uncertain"
"design_conflict"
],
"user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.",
"reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers."
reports/reference-synthesis.md
+22 -22
View File
@@ -2,7 +2,7 @@
Skill: `yao-meta-skill`
- Description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
- Intent confidence: `30/100` (`low`)
- Intent confidence: `100/100` (`high`)
## Live GitHub Benchmarks
@@ -26,72 +26,72 @@ Skill: `yao-meta-skill`
## Curated World-Class Pattern Tracks
### Official workflow product ergonomics
### Official skill anatomy and context discipline
- Type: `official`
- Evidence mode: `curated-pattern-track`
- Why relevant: This track matches: workflow.
- Borrow: Borrow a first-time operator flow that explains itself before it asks for more structure.
- Avoid: Do not mimic product polish that adds UI bulk without improving clarity.
- Why relevant: This track matches: metadata, references.
- Borrow: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.
- Avoid: Do not let packaging or platform concerns swallow the core job boundary.
### Hypothesis-test-learn loop
### Human-in-the-loop verification
- Type: `research`
- Evidence mode: `curated-pattern-track`
- Why relevant: This track matches: general fit.
- Borrow: Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.
- Avoid: Do not create experimental overhead that exceeds the skill's real risk tier.
- Why relevant: This track matches: review, govern.
- Borrow: Borrow a review checkpoint wherever trust matters more than raw speed.
- Avoid: Do not force every skill through heavyweight review when the risk is low.
### Boundary-first design
- Type: `principles`
- Evidence mode: `curated-pattern-track`
- Why relevant: This track matches: general fit.
- Why relevant: This track matches: route.
- Borrow: Borrow the discipline of defining what the skill should not own before growing the package.
- Avoid: Do not expand execution assets until route boundaries stay clean.
## Borrow Now
- Borrow a first-time operator flow that explains itself before it asks for more structure.
- Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.
- Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.
- Borrow a review checkpoint wherever trust matters more than raw speed.
- Borrow the discipline of defining what the skill should not own before growing the package.
- Borrow the way it turns a messy workflow into a repeatable operating path.
- Borrow the clear execution entrypoints and command structure.
## Avoid Now
- Do not mimic product polish that adds UI bulk without improving clarity.
- Do not create experimental overhead that exceeds the skill's real risk tier.
- Do not let packaging or platform concerns swallow the core job boundary.
- Do not force every skill through heavyweight review when the risk is low.
- Do not expand execution assets until route boundaries stay clean.
- Do not import process overhead that only exists for that project's scale.
- Do not copy repo-specific commands or environment assumptions verbatim.
## Pattern Gate
- Summary: 3 accepted, 3 deferred using threshold 4/4.
- Summary: 5 accepted, 1 deferred using threshold 4/4.
- Acceptance threshold: `4/4`
- Accepted patterns:
- **Official workflow product ergonomics**: 4/4 (recurrence, generativity, distinctiveness, boundary)
- **Official skill anatomy and context discipline**: 4/4 (recurrence, generativity, distinctiveness, boundary)
- **Human-in-the-loop verification**: 4/4 (recurrence, generativity, distinctiveness, boundary)
- **Boundary-first design**: 4/4 (recurrence, generativity, distinctiveness, boundary)
- **obra/superpowers**: 4/4 (recurrence, generativity, distinctiveness, boundary)
- **affaan-m/ECC**: 4/4 (recurrence, generativity, distinctiveness, boundary)
- Deferred patterns:
- **Hypothesis-test-learn loop**: missing distinctiveness
- **Boundary-first design**: missing distinctiveness
- **multica-ai/andrej-karpathy-skills**: missing generativity
## Default Recommendation
- Summary: Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.
- Why: Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package.
- Summary: Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary.
- Why: There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.
- User decision required: `True`
## Visibility Mode
- Mode: `explicit`
- Reasons: intent_uncertain
- Reasons: design_conflict
- User note: Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.
- Reviewer note: Keep the full benchmark and synthesis evidence visible for authors and reviewers.
## Conflict Check
- No material design conflict detected. Keep the synthesis silent for the user.
- **lightweight_vs_governance**: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.
## Quality Lift Thesis
reports/registry_audit.json
+91
View File
@@ -0,0 +1,91 @@
{
"ok": true,
"schema_version": "2.0",
"skill_dir": ".",
"registry_dir": "registry",
"package": {
"schema_version": "2.0",
"name": "yao-meta-skill",
"version": "1.1.0",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"targets": [
"openai",
"claude",
"generic",
"agent-skills-compatible"
],
"maturity": "governed",
"owner": "Yao Team",
"review_cadence": "quarterly",
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe",
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06"
},
"compatibility": {
"openai": "pass",
"claude": "pass",
"agent-skills": "pass",
"vscode": "pass",
"generic": "pass",
"agent-skills-compatible": "pass"
},
"source": {
"skill_dir": ".",
"skill_ir": "skill-ir/examples/yao-meta-skill.json",
"ir_schema_version": "2.0.0",
"canonical_metadata": "agents/interface.yaml"
},
"artifacts": {
"overview_html": "reports/skill-overview.html",
"review_studio_html": "reports/review-studio.html",
"trust_report": "reports/security_trust_report.md",
"conformance_matrix": "reports/conformance_matrix.md",
"compiled_targets": "reports/compiled_targets.md",
"package_verification": "reports/package_verification.md",
"install_simulation": "reports/install_simulation.md",
"adoption_drift": "reports/adoption_drift_report.md",
"review_waivers": "reports/review_waivers.md",
"review_annotations": "reports/review_annotations.md",
"package_metadata": "registry/packages/yao-meta-skill.json"
},
"distribution": {
"archive_verified": true,
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
},
"generated_at": "2026-06-13"
},
"index": {
"schema_version": "2.0",
"generated_at": "2026-06-13",
"package_count": 1,
"packages": [
{
"name": "yao-meta-skill",
"version": "1.1.0",
"maturity": "governed",
"owner": "Yao Team",
"targets": [
"openai",
"claude",
"generic",
"agent-skills-compatible"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe"
}
]
},
"failures": [],
"warnings": [],
"artifacts": {
"index": "registry/index.json",
"package": "registry/packages/yao-meta-skill.json",
"json": "reports/registry_audit.json",
"markdown": "reports/registry_audit.md"
}
}
reports/registry_audit.md
+37
View File
@@ -0,0 +1,37 @@
# Registry Audit
- OK: `True`
- Package: `yao-meta-skill`
- Version: `1.1.0`
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe`
- Archive SHA256: `4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06`
- Install simulated: `True`
## Compatibility
| Target | Status |
| --- | --- |
| `openai` | `pass` |
| `claude` | `pass` |
| `agent-skills` | `pass` |
| `vscode` | `pass` |
| `generic` | `pass` |
| `agent-skills-compatible` | `pass` |
## Failures
- None
## Warnings
- None
## Artifacts
- index: `registry/index.json`
- package: `registry/packages/yao-meta-skill.json`
- json: `reports/registry_audit.json`
- markdown: `reports/registry_audit.md`
reports/review-studio.html
+155 -18
View File
@@ -116,8 +116,89 @@
gap: 3px;
}
.muted { color: var(--muted); }
.actions-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 16px;
}
.annotations-grid {
display: grid;
grid-template-columns: repeat(2, minmax(0, 1fr));
gap: 16px;
}
.action-card {
border: 1px solid var(--line);
border-radius: 8px;
padding: 18px;
background: #fff;
display: grid;
gap: 10px;
min-width: 0;
}
.action-card.warn { border-left: 4px solid var(--warn); }
.action-card.block { border-left: 4px solid var(--block); }
.annotation-card {
border: 1px solid var(--line);
border-radius: 8px;
padding: 18px;
background: #fff;
display: grid;
gap: 10px;
min-width: 0;
}
.annotation-card.warning { border-left: 4px solid var(--warn); }
.annotation-card.blocker { border-left: 4px solid var(--block); }
.annotation-card.resolved { opacity: .72; }
.action-card span,
.annotation-card span,
.action-card small,
.annotation-card small,
.action-card footer,
.annotation-card footer,
.action-card dd {
color: var(--muted);
font-size: 14px;
overflow-wrap: anywhere;
}
.annotation-card dd { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
.action-card dl, .annotation-card dl {
display: grid;
grid-template-columns: 80px minmax(0, 1fr);
gap: 6px 10px;
margin: 0;
}
.action-card dt, .annotation-card dt { color: var(--ink); font-size: 14px; }
.action-card dd, .annotation-card dd { margin: 0; }
.source-ref-list {
list-style: none;
padding: 0;
margin: 0;
display: grid;
gap: 6px;
}
.source-ref-list li {
display: grid;
gap: 2px;
min-width: 0;
padding: 8px 0 0;
border-top: 1px solid var(--line);
}
.source-ref-list a,
.source-ref-list span {
overflow-wrap: anywhere;
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 13px;
}
.source-ref-list small {
font-size: 12px;
color: var(--muted);
}
code {
font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
font-size: 13px;
}
@media (max-width: 980px) {
.metrics, .gates, .twocol { grid-template-columns: 1fr; }
.metrics, .gates, .twocol, .actions-grid, .annotations-grid { grid-template-columns: 1fr; }
main { padding: 32px 18px 60px; }
nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; }
nav a { flex: 0 0 auto; }
@@ -125,7 +206,7 @@
</style>
</head>
<body>
<nav><a href='#overview'>审查总览</a><a href='#intent'>意图画布</a><a href='#trigger'>触发实验</a><a href='#output'>输出实验</a><a href='#runtime'>运行矩阵</a><a href='#trust'>信任报告</a><a href='#atlas'>组合治理</a><a href='#release'>发布路线</a></nav>
<nav><a href='#overview'>审查总览</a><a href='#intent'>意图画布</a><a href='#trigger'>触发实验</a><a href='#output'>输出实验</a><a href='#actions'>修复动作</a><a href='#annotations'>审查批注</a><a href='#runtime'>运行矩阵</a><a href='#trust'>信任报告</a><a href='#permissions'>权限批准</a><a href='#permission-probes'>权限探针</a><a href='#atlas'>组合治理</a><a href='#telemetry'>运营回路</a><a href='#waivers'>人工批准</a><a href='#registry'>注册审计</a><a href='#release'>发布路线</a></nav>
<main>
<header id="overview">
<div class="eyebrow">Review Studio 2.0</div>
@@ -133,65 +214,121 @@
<p class="lede">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</p>
<div class="decision">
<span>审查结论</span>
<strong>review</strong>
<span>Score 87/100</span>
<strong>ready</strong>
<span>Score 100/100</span>
</div>
</header>
<section>
<h2>核心指标</h2>
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>4 targets in platform-neutral contract</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>50 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article></div>
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>4 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>4/4</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>3/3</strong><p>0 native enforcement targets</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>62 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>4 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>483 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>3 adapters readable; local install simulation</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
</section>
<section>
<h2>审查闸门</h2>
<div class="gates"><article class='gate warn'><div><span>关注</span><h3>意图画布</h3></div><p>intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 910/1000; quality density 142.9</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>信任报告</h3></div><p>0 secrets; 50 scripts; 2 network-capable scripts</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>组合治理</h3></div><p>12 skills; 5 route collisions; 9 owner gaps; 10 stale</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked</p><footer>reports/promotion_decisions.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 987/1000; quality density 131.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 62 scripts; 2 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>3/3 targets probed; native 0; metadata fallback 3; residual risks 3</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>人工批准</h3></div><p>0 active waivers cover current warnings</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
</section>
<div class="twocol">
<section><h2>阻断事项</h2><p class='muted'>无。</p></section>
<section><h2>关注事项</h2><ul class='issues'><li><strong>意图画布</strong><span>intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.</span></li><li><strong>信任报告</strong><span>0 secrets; 50 scripts; 2 network-capable scripts</span></li><li><strong>组合治理</strong><span>12 skills; 5 route collisions; 9 owner gaps; 10 stale</span></li></ul></section>
<section><h2>关注事项</h2><p class='muted'>无。</p></section>
</div>
<section id="actions">
<h2>修复动作</h2>
<div class="actions-grid"><p class='muted'>当前没有 blocker 或 warning。保持现有证据链即可。</p></div>
</section>
<section id="annotations">
<h2>审查批注</h2>
<p class="muted">当前批注：0 条批注；开放 0；阻断 0</p>
<div class="annotations-grid"><p class='muted'>当前没有 reviewer 批注。</p></div>
</section>
<section id="intent" class="twocol">
<div class="panel">
<h2>意图画布</h2>
<p>Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</p>
<p>Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.</p>
</div>
<div class="panel">
<h2>证据路径</h2>
<ul class="evidence"><li><strong>skill_overview</strong><span>reports/skill-overview.html</span></li><li><strong>review_viewer</strong><span>reports/review-viewer.html</span></li><li><strong>output_eval</strong><span>reports/output_quality_scorecard.md</span></li><li><strong>runtime_conformance</strong><span>reports/conformance_matrix.md</span></li><li><strong>trust_report</strong><span>reports/security_trust_report.md</span></li><li><strong>skill_atlas</strong><span>reports/skill_atlas.html</span></li><li><strong>migration</strong><span>docs/migration-v2.md</span></li></ul>
<ul class="evidence"><li><strong>skill_overview</strong><span>reports/skill-overview.html</span></li><li><strong>review_viewer</strong><span>reports/review-viewer.html</span></li><li><strong>output_eval</strong><span>reports/output_quality_scorecard.md</span></li><li><strong>output_execution</strong><span>reports/output_execution_runs.md</span></li><li><strong>output_blind_review</strong><span>reports/output_blind_review_pack.md</span></li><li><strong>output_review_adjudication</strong><span>reports/output_review_adjudication.md</span></li><li><strong>runtime_conformance</strong><span>reports/conformance_matrix.md</span></li><li><strong>trust_report</strong><span>reports/security_trust_report.md</span></li><li><strong>permission_policy</strong><span>security/permission_policy.md</span></li><li><strong>runtime_permissions</strong><span>reports/runtime_permission_probes.md</span></li><li><strong>skill_atlas</strong><span>reports/skill_atlas.html</span></li><li><strong>compiled_targets</strong><span>reports/compiled_targets.md</span></li><li><strong>adoption_drift</strong><span>reports/adoption_drift_report.md</span></li><li><strong>review_waivers</strong><span>reports/review_waivers.md</span></li><li><strong>review_annotations</strong><span>reports/review_annotations.md</span></li><li><strong>registry_audit</strong><span>reports/registry_audit.md</span></li><li><strong>package_verification</strong><span>reports/package_verification.md</span></li><li><strong>install_simulation</strong><span>reports/install_simulation.md</span></li><li><strong>upgrade_check</strong><span>reports/upgrade_check.md</span></li><li><strong>migration</strong><span>docs/migration-v2.md</span></li></ul>
</div>
</section>
<section id="trigger" class="twocol">
<div class="panel"><h2>触发实验</h2><p>13 trigger cases; 0 misroutes; 0 ambiguous</p></div>
<div class="panel"><h2>组合治理</h2><p>{&#x27;skill_count&#x27;: 12, &#x27;route_collision_count&#x27;: 5, &#x27;owner_gap_count&#x27;: 9, &#x27;stale_count&#x27;: 10, &#x27;shared_resource_count&#x27;: 0, &#x27;no_route_opportunity_count&#x27;: 0}</p></div>
<div class="panel"><h2>组合治理</h2><p>{&#x27;skill_count&#x27;: 12, &#x27;actionable_skill_count&#x27;: 1, &#x27;route_collision_count&#x27;: 5, &#x27;actionable_route_collision_count&#x27;: 0, &#x27;owner_gap_count&#x27;: 9, &#x27;actionable_owner_gap_count&#x27;: 0, &#x27;stale_count&#x27;: 10, &#x27;actionable_stale_count&#x27;: 0, &#x27;shared_resource_count&#x27;: 0, &#x27;no_route_opportunity_count&#x27;: 0, &#x27;non_actionable_issue_count&#x27;: 24}</p></div>
</section>
<section id="output" class="twocol">
<div class="panel"><h2>输出实验</h2><p>{&#x27;case_count&#x27;: 5, &#x27;file_backed_case_count&#x27;: 1, &#x27;near_neighbor_case_count&#x27;: 1, &#x27;boundary_case_count&#x27;: 1, &#x27;baseline_pass_rate&#x27;: 0.0, &#x27;with_skill_pass_rate&#x27;: 100.0, &#x27;delta&#x27;: 100.0, &#x27;regression_count&#x27;: 0, &#x27;gate_pass&#x27;: True, &#x27;failure_taxonomy&#x27;: []}</p></div>
<div class="panel"><h2>发布标准</h2><p>Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor 和 boundary case。</p></div>
<div class="panel"><h2>输出实验</h2><p>{&#x27;case_count&#x27;: 5, &#x27;file_backed_case_count&#x27;: 1, &#x27;near_neighbor_case_count&#x27;: 1, &#x27;boundary_case_count&#x27;: 1, &#x27;baseline_pass_rate&#x27;: 0.0, &#x27;with_skill_pass_rate&#x27;: 100.0, &#x27;delta&#x27;: 100.0, &#x27;regression_count&#x27;: 0, &#x27;gate_pass&#x27;: True, &#x27;failure_taxonomy&#x27;: [], &#x27;blind_pair_count&#x27;: 5}</p></div>
<div class="panel"><h2>执行证据</h2><p>{&#x27;case_count&#x27;: 5, &#x27;variant_run_count&#x27;: 10, &#x27;command_executed_count&#x27;: 10, &#x27;model_executed_count&#x27;: 0, &#x27;recorded_fixture_count&#x27;: 0, &#x27;timing_observed_count&#x27;: 10, &#x27;token_observed_count&#x27;: 0, &#x27;token_estimated_count&#x27;: 10, &#x27;baseline_pass_rate&#x27;: 0.0, &#x27;with_skill_pass_rate&#x27;: 100.0, &#x27;delta&#x27;: 100.0, &#x27;regression_count&#x27;: 0, &#x27;failure_count&#x27;: 0, &#x27;gate_pass&#x27;: True}</p></div>
</section>
<section class="twocol">
<div class="panel"><h2>盲评包</h2><p>{&#x27;pair_count&#x27;: 5, &#x27;answer_key_separate&#x27;: True, &#x27;with_skill_hidden_count&#x27;: 5}</p></div>
<div class="panel"><h2>审定报告</h2><p>{&#x27;pair_count&#x27;: 5, &#x27;judgment_count&#x27;: 0, &#x27;pending_count&#x27;: 5, &#x27;agreement_count&#x27;: 0, &#x27;disagreement_count&#x27;: 0, &#x27;invalid_decision_count&#x27;: 0, &#x27;agreement_rate&#x27;: None, &#x27;needs_review&#x27;: True, &#x27;failure_count&#x27;: 0}</p></div>
</section>
<section class="twocol">
<div class="panel"><h2>发布标准</h2><p>Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。</p></div>
<div class="panel"><h2>人工结论</h2><p>没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。</p></div>
</section>
<section class="twocol">
<div class="panel"><h2>评审方式</h2><p>先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。</p></div>
<div class="panel"><h2>运行方式</h2><p>reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。</p></div>
</section>
<section id="runtime" class="twocol">
<div class="panel"><h2>运行矩阵</h2><p>{&#x27;target_count&#x27;: 5, &#x27;pass_count&#x27;: 5, &#x27;fail_count&#x27;: 0}</p></div>
<div class="panel"><h2>上下文</h2><p>initial load 910/1000; quality density 142.9</p></div>
<div class="panel"><h2>目标编译</h2><p>{&#x27;target_count&#x27;: 4, &#x27;pass_count&#x27;: 4, &#x27;warn_count&#x27;: 0, &#x27;block_count&#x27;: 0, &#x27;failure_count&#x27;: 0, &#x27;warning_count&#x27;: 0}</p></div>
</section>
<section class="twocol">
<div class="panel"><h2>上下文</h2><p>initial load 987/1000; quality density 131.7</p></div>
<div class="panel"><h2>编译证据</h2><p>Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.</p></div>
</section>
<section id="trust" class="twocol">
<div class="panel"><h2>信任报告</h2><p>{&#x27;scanned_files&#x27;: 127, &#x27;script_count&#x27;: 50, &#x27;secret_findings&#x27;: 0, &#x27;dependency_files&#x27;: [&#x27;requirements-ci.txt&#x27;], &#x27;network_script_count&#x27;: 2, &#x27;interactive_script_count&#x27;: 0, &#x27;package_sha256&#x27;: &#x27;18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad&#x27;}</p></div>
<div class="panel"><h2>信任报告</h2><p>{&#x27;scanned_files&#x27;: 146, &#x27;script_count&#x27;: 62, &#x27;internal_module_count&#x27;: 3, &#x27;secret_findings&#x27;: 0, &#x27;dependency_files&#x27;: [&#x27;requirements-ci.txt&#x27;], &#x27;network_script_count&#x27;: 2, &#x27;network_policy_covered_count&#x27;: 2, &#x27;network_policy_missing_count&#x27;: 0, &#x27;file_write_script_count&#x27;: 47, &#x27;permission_required_count&#x27;: 3, &#x27;permission_approved_count&#x27;: 3, &#x27;permission_missing_count&#x27;: 0, &#x27;permission_invalid_count&#x27;: 0, &#x27;permission_expired_count&#x27;: 0, &#x27;help_smoke_checked_count&#x27;: 59, &#x27;help_smoke_failed_count&#x27;: 0, &#x27;interactive_script_count&#x27;: 0, &#x27;package_hash_scope&#x27;: &#x27;source-contract-without-generated-reports&#x27;, &#x27;package_hash_file_count&#x27;: 146, &#x27;package_sha256&#x27;: &#x27;9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe&#x27;}</p></div>
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
</section>
<section id="permissions" class="twocol">
<div class="panel"><h2>权限批准</h2><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p></div>
<div class="panel"><h2>批准策略</h2><p>高权限能力需要 reviewer、scope、reason、expires_at 和 openai/claude/generic 目标端 enforcement 说明。</p></div>
</section>
<section id="permission-probes" class="twocol">
<div class="panel"><h2>权限探针</h2><p>3/3 targets probed; native 0; metadata fallback 3; residual risks 3</p></div>
<div class="panel"><h2>运行边界</h2><p>{&#x27;target_count&#x27;: 3, &#x27;pass_count&#x27;: 3, &#x27;fail_count&#x27;: 0, &#x27;native_enforcement_count&#x27;: 0, &#x27;metadata_fallback_count&#x27;: 3, &#x27;residual_risk_count&#x27;: 3, &#x27;required_capability_count&#x27;: 3, &#x27;failure_count&#x27;: 0}</p></div>
</section>
<section id="atlas" class="twocol">
<div class="panel"><h2>组合治理</h2><p>12 skills; 5 route collisions; 9 owner gaps; 10 stale</p></div>
<div class="panel"><h2>下一动作</h2><p>优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再做 registry 或 telemetry。</p></div>
<div class="panel"><h2>组合治理</h2><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues</p></div>
<div class="panel"><h2>下一动作</h2><p>优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再用运营回路判断真实影响。</p></div>
</section>
<section id="telemetry" class="twocol">
<div class="panel"><h2>运营回路</h2><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low</p></div>
<div class="panel"><h2>漂移信号</h2><p>{&#x27;event_count&#x27;: 1, &#x27;activation_count&#x27;: 1, &#x27;accepted_count&#x27;: 1, &#x27;edited_count&#x27;: 0, &#x27;rejected_count&#x27;: 0, &#x27;missed_count&#x27;: 0, &#x27;failed_count&#x27;: 0, &#x27;adoption_rate&#x27;: 100.0, &#x27;missed_trigger_count&#x27;: 0, &#x27;wrong_trigger_count&#x27;: 0, &#x27;bad_output_count&#x27;: 0, &#x27;script_error_count&#x27;: 0, &#x27;missing_resource_count&#x27;: 0, &#x27;review_overdue_count&#x27;: 0, &#x27;risk_band&#x27;: &#x27;low&#x27;, &#x27;event_types&#x27;: {&#x27;skill_activation&#x27;: 1}, &#x27;failure_types&#x27;: {}}</p></div>
</section>
<section id="waivers" class="twocol">
<div class="panel"><h2>人工批准</h2><p>0 active waivers cover current warnings</p></div>
<div class="panel"><h2>批准台账</h2><p>{&#x27;waiver_count&#x27;: 0, &#x27;active_count&#x27;: 0, &#x27;expired_count&#x27;: 0, &#x27;invalid_count&#x27;: 0, &#x27;covered_gate_count&#x27;: 0, &#x27;covered_gate_keys&#x27;: []}</p></div>
</section>
<section id="registry" class="twocol">
<div class="panel"><h2>注册审计</h2><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters</p></div>
<div class="panel"><h2>包体元数据</h2><p>{&#x27;schema_version&#x27;: &#x27;2.0&#x27;, &#x27;name&#x27;: &#x27;yao-meta-skill&#x27;, &#x27;version&#x27;: &#x27;1.1.0&#x27;, &#x27;description&#x27;: &#x27;Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.&#x27;, &#x27;targets&#x27;: [&#x27;openai&#x27;, &#x27;claude&#x27;, &#x27;generic&#x27;, &#x27;agent-skills-compatible&#x27;], &#x27;maturity&#x27;: &#x27;governed&#x27;, &#x27;owner&#x27;: &#x27;Yao Team&#x27;, &#x27;review_cadence&#x27;: &#x27;quarterly&#x27;, &#x27;trust_level&#x27;: &#x27;local&#x27;, &#x27;license&#x27;: &#x27;MIT&#x27;, &#x27;checksums&#x27;: {&#x27;package_sha256&#x27;: &#x27;9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe&#x27;, &#x27;archive_sha256&#x27;: &#x27;4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06&#x27;}, &#x27;compatibility&#x27;: {&#x27;openai&#x27;: &#x27;pass&#x27;, &#x27;claude&#x27;: &#x27;pass&#x27;, &#x27;agent-skills&#x27;: &#x27;pass&#x27;, &#x27;vscode&#x27;: &#x27;pass&#x27;, &#x27;generic&#x27;: &#x27;pass&#x27;, &#x27;agent-skills-compatible&#x27;: &#x27;pass&#x27;}, &#x27;source&#x27;: {&#x27;skill_dir&#x27;: &#x27;.&#x27;, &#x27;skill_ir&#x27;: &#x27;skill-ir/examples/yao-meta-skill.json&#x27;, &#x27;ir_schema_version&#x27;: &#x27;2.0.0&#x27;, &#x27;canonical_metadata&#x27;: &#x27;agents/interface.yaml&#x27;}, &#x27;artifacts&#x27;: {&#x27;overview_html&#x27;: &#x27;reports/skill-overview.html&#x27;, &#x27;review_studio_html&#x27;: &#x27;reports/review-studio.html&#x27;, &#x27;trust_report&#x27;: &#x27;reports/security_trust_report.md&#x27;, &#x27;conformance_matrix&#x27;: &#x27;reports/conformance_matrix.md&#x27;, &#x27;compiled_targets&#x27;: &#x27;reports/compiled_targets.md&#x27;, &#x27;package_verification&#x27;: &#x27;reports/package_verification.md&#x27;, &#x27;install_simulation&#x27;: &#x27;reports/install_simulation.md&#x27;, &#x27;adoption_drift&#x27;: &#x27;reports/adoption_drift_report.md&#x27;, &#x27;review_waivers&#x27;: &#x27;reports/review_waivers.md&#x27;, &#x27;review_annotations&#x27;: &#x27;reports/review_annotations.md&#x27;, &#x27;package_metadata&#x27;: &#x27;registry/packages/yao-meta-skill.json&#x27;}, &#x27;distribution&#x27;: {&#x27;archive_verified&#x27;: True, &#x27;archive_sha256&#x27;: &#x27;4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06&#x27;, &#x27;package_verification&#x27;: &#x27;reports/package_verification.json&#x27;, &#x27;install_simulated&#x27;: True, &#x27;install_simulation&#x27;: &#x27;reports/install_simulation.json&#x27;}, &#x27;generated_at&#x27;: &#x27;2026-06-13&#x27;}</p></div>
</section>
<section id="release" class="twocol">
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked</p></div>
<div class="panel"><h2>世界级缺口</h2><p>下一阶段应继续推进 IR-first compiler、registry audit、telemetry drift loop，以及更严格的 governed trust gates。</p></div>
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p></div>
<div class="panel"><h2>包体验证</h2><p>{&#x27;target_count&#x27;: 3, &#x27;adapter_count&#x27;: 3, &#x27;archive_present&#x27;: True, &#x27;archive_sha256&#x27;: &#x27;4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06&#x27;, &#x27;archive_entry_count&#x27;: 483, &#x27;failure_count&#x27;: 0, &#x27;warning_count&#x27;: 0}</p></div>
</section>
</main>
</body>
reports/review-studio.json
+7106 -301
View File
File diff suppressed because it is too large Load Diff
reports/review_annotations.json
+24
View File
@@ -0,0 +1,24 @@
{
"schema_version": "1.0",
"ok": true,
"skill_dir": ".",
"source": "tests/tmp_review_studio/empty_review_annotations_input.json",
"summary": {
"annotation_count": 0,
"open_count": 0,
"resolved_count": 0,
"deferred_count": 0,
"open_blocker_count": 0,
"open_warning_count": 0,
"linked_gate_count": 0,
"target_missing_count": 0,
"failure_count": 0
},
"annotations": [],
"failures": [],
"template_written": false,
"artifacts": {
"json": "reports/review_annotations.json",
"markdown": "reports/review_annotations.md"
}
}
reports/review_annotations.md
+19
View File
@@ -0,0 +1,19 @@
# Review Annotations
This report renders reviewer annotations attached to Review Studio gates and source/report paths.
- Annotations: `0`
- Open: `0`
- Resolved: `0`
- Deferred: `0`
- Open blockers: `0`
- Open warnings: `0`
No reviewer annotations recorded yet.
## Review Rule
- Use annotations for reviewer comments tied to a gate or source line.
- Use waivers only for explicit acceptance of warning-level release risk.
- Open blocker annotations should block a release decision until resolved or deferred with rationale.
reports/review_waivers.json
+40
View File
@@ -0,0 +1,40 @@
{
"schema_version": "1.0",
"ok": true,
"skill_dir": ".",
"generated_at": "2026-06-13",
"summary": {
"waiver_count": 0,
"active_count": 0,
"expired_count": 0,
"invalid_count": 0,
"covered_gate_count": 0,
"covered_gate_keys": []
},
"policy": {
"blocker_waivers_allowed": false,
"minimum_reason_chars": 20,
"expires_required": true,
"known_gate_keys": [
"context-budget",
"intent-canvas",
"operations-loop",
"output-lab",
"permission-gates",
"permission-runtime",
"registry-audit",
"release-notes",
"runtime-matrix",
"skill-atlas",
"trigger-lab",
"trust-report"
]
},
"waivers": [],
"failures": [],
"warnings": [],
"artifacts": {
"json": "reports/review_waivers.json",
"markdown": "reports/review_waivers.md"
}
}
reports/review_waivers.md
+26
View File
@@ -0,0 +1,26 @@
# Review Waivers
- OK: `True`
- Waivers: `0`
- Active: `0`
- Expired: `0`
- Invalid: `0`
- Covered gates: `none`
## Policy
- Blocker waivers allowed: `False`
- Minimum reason chars: `20`
- Expiry is required for every waiver.
## Waivers
- None
## Failures
- None
## Warnings
- None
reports/runtime_permission_probes.json
+253
View File
@@ -0,0 +1,253 @@
{
"schema_version": "1.0",
"ok": true,
"skill_dir": ".",
"package_dir": "tests/tmp_review_studio/dist",
"expected_capabilities": [
"file_write",
"network",
"subprocess"
],
"summary": {
"target_count": 3,
"pass_count": 3,
"fail_count": 0,
"native_enforcement_count": 0,
"metadata_fallback_count": 3,
"residual_risk_count": 3,
"required_capability_count": 3,
"failure_count": 0
},
"targets": [
{
"target": "openai",
"status": "pass",
"adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json",
"permission_model": "metadata-only",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"assurance": "metadata-fallback-explicit",
"declared_capabilities": [
"file_write",
"network",
"subprocess"
],
"checks": [
{
"key": "adapter-present",
"passed": true,
"detail": "openai adapter.json is readable"
},
{
"key": "permission-contract-present",
"passed": true,
"detail": "openai adapter includes permission_contract"
},
{
"key": "target-contract-present",
"passed": true,
"detail": "openai adapter includes target_permission_contract"
},
{
"key": "source-available",
"passed": true,
"detail": "openai permission_contract links to an available trust report"
},
{
"key": "declared-capabilities-match",
"passed": true,
"detail": "openai target_permission_contract mirrors expected capabilities"
},
{
"key": "capability-counts-present",
"passed": true,
"detail": "openai target_permission_contract includes capability_counts"
},
{
"key": "native-enforcement-boolean",
"passed": true,
"detail": "openai target_permission_contract declares native_enforcement as a boolean"
},
{
"key": "representation-present",
"passed": true,
"detail": "openai target_permission_contract declares where permission metadata is represented"
},
{
"key": "operator-note-present",
"passed": true,
"detail": "openai target_permission_contract includes an operator_note"
},
{
"key": "review-required-matches",
"passed": true,
"detail": "openai review_required matches whether capabilities are required"
},
{
"key": "openai-yaml-present",
"passed": true,
"detail": "OpenAI permission metadata YAML is readable"
},
{
"key": "openai-yaml-permissions",
"passed": true,
"detail": "OpenAI YAML permission contract mirrors expected capabilities"
},
{
"key": "openai-yaml-native-flag",
"passed": true,
"detail": "OpenAI YAML declares native_enforcement as a boolean"
}
],
"failures": [],
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
},
{
"target": "claude",
"status": "pass",
"adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json",
"permission_model": "neutral-source-plus-adapter",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"assurance": "metadata-fallback-explicit",
"declared_capabilities": [
"file_write",
"network",
"subprocess"
],
"checks": [
{
"key": "adapter-present",
"passed": true,
"detail": "claude adapter.json is readable"
},
{
"key": "permission-contract-present",
"passed": true,
"detail": "claude adapter includes permission_contract"
},
{
"key": "target-contract-present",
"passed": true,
"detail": "claude adapter includes target_permission_contract"
},
{
"key": "source-available",
"passed": true,
"detail": "claude permission_contract links to an available trust report"
},
{
"key": "declared-capabilities-match",
"passed": true,
"detail": "claude target_permission_contract mirrors expected capabilities"
},
{
"key": "capability-counts-present",
"passed": true,
"detail": "claude target_permission_contract includes capability_counts"
},
{
"key": "native-enforcement-boolean",
"passed": true,
"detail": "claude target_permission_contract declares native_enforcement as a boolean"
},
{
"key": "representation-present",
"passed": true,
"detail": "claude target_permission_contract declares where permission metadata is represented"
},
{
"key": "operator-note-present",
"passed": true,
"detail": "claude target_permission_contract includes an operator_note"
},
{
"key": "review-required-matches",
"passed": true,
"detail": "claude review_required matches whether capabilities are required"
}
],
"failures": [],
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
},
{
"target": "generic",
"status": "pass",
"adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json",
"permission_model": "agent-skills-compatible-metadata",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"assurance": "metadata-fallback-explicit",
"declared_capabilities": [
"file_write",
"network",
"subprocess"
],
"checks": [
{
"key": "adapter-present",
"passed": true,
"detail": "generic adapter.json is readable"
},
{
"key": "permission-contract-present",
"passed": true,
"detail": "generic adapter includes permission_contract"
},
{
"key": "target-contract-present",
"passed": true,
"detail": "generic adapter includes target_permission_contract"
},
{
"key": "source-available",
"passed": true,
"detail": "generic permission_contract links to an available trust report"
},
{
"key": "declared-capabilities-match",
"passed": true,
"detail": "generic target_permission_contract mirrors expected capabilities"
},
{
"key": "capability-counts-present",
"passed": true,
"detail": "generic target_permission_contract includes capability_counts"
},
{
"key": "native-enforcement-boolean",
"passed": true,
"detail": "generic target_permission_contract declares native_enforcement as a boolean"
},
{
"key": "representation-present",
"passed": true,
"detail": "generic target_permission_contract declares where permission metadata is represented"
},
{
"key": "operator-note-present",
"passed": true,
"detail": "generic target_permission_contract includes an operator_note"
},
{
"key": "review-required-matches",
"passed": true,
"detail": "generic review_required matches whether capabilities are required"
}
],
"failures": [],
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
}
],
"failures": [],
"artifacts": {
"json": "reports/runtime_permission_probes.json",
"markdown": "reports/runtime_permission_probes.md"
}
}
reports/runtime_permission_probes.md
+27
View File
@@ -0,0 +1,27 @@
# Runtime Permission Probes
Runtime permission probes verify that generated target adapters expose high-permission capabilities and make native-enforcement limits explicit.
## Summary
- OK: `True`
- Targets probed: `3`
- Passed: `3`
- Failed: `0`
- Native enforcement targets: `0`
- Explicit metadata fallbacks: `3`
- Required capabilities: `file_write, network, subprocess`
| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Residual Risk |
| --- | --- | --- | --- | --- | --- |
| `openai` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. |
| `claude` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. |
| `generic` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. |
## Failures
- None
## Reviewer Note
A passing probe means the target contract is explicit and auditable. It does not claim that a host client enforces permissions natively.
reports/security_trust_report.json
+1327 -60
View File
File diff suppressed because it is too large Load Diff
reports/security_trust_report.md
+104 -57
View File
@@ -1,12 +1,22 @@
# Security Trust Report
- OK: `True`
- Scanned files: `127`
- Scripts: `50`
- Scanned files: `146`
- Scripts: `62`
- Internal script modules: `3`
- Secret findings: `0`
- Network-capable scripts: `2`
- Network policy covered scripts: `2`
- Network policy missing scripts: `0`
- File-write scripts: `47`
- Permission approvals: `3 / 3`
- Permission approval gaps: `0`
- CLI help smoke checked: `59`
- CLI help smoke failures: `0`
- Interactive scripts: `0`
- Package SHA256: `18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `146`
- Package SHA256: `9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe`
## Failures
@@ -14,8 +24,7 @@
## Warnings
- Scripts without argparse/help surface: scripts/render_context_reports.py, scripts/render_social_preview.py, scripts/skill_report_charts.py, scripts/skill_report_metrics.py, scripts/skill_report_model.py
- Network-capable scripts require bounded host policy: scripts/check_update.py, scripts/github_benchmark_scan.py
- None
## Dependency Evidence
@@ -23,57 +32,95 @@
- Pinned entries: `1`
- Unpinned entries: `0`
## Network Policy
- Policy file: `security/network_policy.json`
- Present: `True`
- Covered scripts: `2`
- Missing scripts: `none`
- Mismatches: `0`
## Permission Governance
- Policy file: `security/permission_policy.json`
- Present: `True`
- Required capabilities: `file_write, network, subprocess`
- Approved capabilities: `file_write, network, subprocess`
- Missing approvals: `none`
- Invalid approvals: `none`
- Expired approvals: `none`
## CLI Help Smoke
- Enabled: `True`
- Timeout seconds: `5.0`
- Checked scripts: `59`
- Passed scripts: `59`
- Failed scripts: `none`
## Script Surface
| Script | Argparse | Main Guard | Input | Network | Subprocess |
| --- | --- | --- | --- | --- | --- |
| scripts/build_confusion_matrix.py | True | True | False | False | False |
| scripts/build_skill_atlas.py | True | True | False | False | False |
| scripts/check_update.py | True | True | False | True | False |
| scripts/ci_test.py | True | True | False | False | True |
| scripts/collect_feedback.py | True | True | False | False | False |
| scripts/context_sizer.py | True | True | False | False | False |
| scripts/create_iteration_snapshot.py | True | True | False | False | False |
| scripts/cross_packager.py | True | True | False | False | False |
| scripts/diff_eval.py | True | True | False | False | False |
| scripts/export_skill_ir.py | True | True | False | False | False |
| scripts/github_benchmark_scan.py | True | True | False | True | False |
| scripts/governance_check.py | True | True | False | False | False |
| scripts/init_skill.py | True | True | False | False | False |
| scripts/judge_blind_eval.py | True | True | False | False | False |
| scripts/lint_skill.py | True | True | False | False | False |
| scripts/optimize_description.py | True | True | False | False | False |
| scripts/promotion_checker.py | True | True | False | False | False |
| scripts/render_artifact_design_profile.py | True | True | False | False | False |
| scripts/render_baseline_compare.py | True | True | False | False | False |
| scripts/render_context_reports.py | False | True | False | False | False |
| scripts/render_description_drift_history.py | True | True | False | False | False |
| scripts/render_eval_dashboard.py | True | True | False | False | True |
| scripts/render_intent_confidence.py | True | True | False | False | False |
| scripts/render_intent_dialogue.py | True | True | False | False | False |
| scripts/render_iteration_directions.py | True | True | False | False | False |
| scripts/render_iteration_ledger.py | True | True | False | False | False |
| scripts/render_output_risk_profile.py | True | True | False | False | False |
| scripts/render_portability_report.py | True | True | False | False | False |
| scripts/render_prompt_quality_profile.py | True | True | False | False | False |
| scripts/render_reference_scan.py | True | True | False | False | False |
| scripts/render_reference_synthesis.py | True | True | False | False | False |
| scripts/render_regression_history.py | True | True | False | False | False |
| scripts/render_review_studio.py | True | True | False | False | False |
| scripts/render_review_viewer.py | True | True | False | False | False |
| scripts/render_skill_overview.py | True | True | False | False | False |
| scripts/render_social_preview.py | False | True | False | False | False |
| scripts/render_system_model.py | True | True | False | False | False |
| scripts/resource_boundary_check.py | True | True | False | False | False |
| scripts/run_conformance_suite.py | True | True | False | False | False |
| scripts/run_description_optimization_suite.py | True | True | False | False | False |
| scripts/run_eval_suite.py | True | True | False | False | True |
| scripts/run_output_eval.py | True | True | False | False | False |
| scripts/skill_report_charts.py | False | False | False | False | False |
| scripts/skill_report_metrics.py | False | False | False | False | False |
| scripts/skill_report_model.py | False | False | False | False | False |
| scripts/sync_local_install.py | True | True | False | False | True |
| scripts/trigger_eval.py | True | True | False | False | False |
| scripts/trust_check.py | True | True | False | False | False |
| scripts/validate_skill.py | True | True | False | False | False |
| scripts/yao.py | True | True | False | False | True |
| Script | Interface | Declared | Argparse | Main Guard | Input | Network | File Write | Subprocess | Reason |
| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
| scripts/adjudicate_output_review.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/build_confusion_matrix.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/build_skill_atlas.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/check_update.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/ci_test.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/collect_feedback.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/compile_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/context_sizer.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/create_iteration_snapshot.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/cross_packager.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/diff_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/export_skill_ir.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/github_benchmark_scan.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/governance_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/init_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/judge_blind_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/lint_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/local_output_eval_runner.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/optimize_description.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/probe_runtime_permissions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/promotion_checker.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/registry_audit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_adoption_drift_report.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_artifact_design_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_baseline_compare.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_context_reports.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_description_drift_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_eval_dashboard.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_intent_confidence.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_intent_dialogue.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_iteration_directions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_iteration_ledger.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_output_risk_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_portability_report.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_prompt_quality_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_reference_scan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_reference_synthesis.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_regression_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_review_annotations.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_review_studio.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_review_viewer.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_review_waivers.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_skill_overview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_social_preview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_system_model.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/run_conformance_suite.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/run_description_optimization_suite.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/run_eval_suite.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/run_output_eval.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/run_output_execution.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/simulate_install.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/skill_report_charts.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to render inline SVG report charts. |
| scripts/skill_report_metrics.py | internal-module | True | False | False | False | False | False | False | Imported by skill_report_model.py to calculate overview report metrics. |
| scripts/skill_report_model.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to build the v2 report data model. |
| scripts/sync_local_install.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/trigger_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/trust_check.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/upgrade_check.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/validate_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/yao.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
reports/skill-os-2-review.md
+70 -36
View File
@@ -8,80 +8,114 @@ Scope: Yao Meta Skill against the user-provided Skill OS 2.0 upgrade plan.
Yao Meta Skill is no longer only a Meta Skill factory. The current working tree now has the first verifiable Skill OS foundation:
- Skill IR v0 for platform-neutral contract capture.
- Output Eval Lab v0 for with-skill vs baseline assertion grading.
- Output Eval Lab v0 for with-skill vs baseline assertion grading plus local command-runner execution evidence, blind A/B review pack, separate answer key evidence, and reviewer adjudication reporting.
- Runtime Conformance v0 for target-consumption checks.
- Trust/Security v0 for secret, dependency, script, trust metadata, and package-integrity checks.
- Trust/Security v0 for secret, dependency, script, bounded network host policy, execution-level CLI help smoke checks, trust metadata, and stable source-contract integrity checks.
- Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, and no-route opportunities.
- Bilingual Skill Overview v2 that includes these evidence surfaces.
- Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, and release-gate review.
- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, parity checks, and IR provenance.
- Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review.
- Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers.
- Target Compiler v0 so Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts before packaging, including target-specific permission contracts and native behavior contracts.
- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, compiler contract, parity checks, and IR provenance.
- Registry & Distribution v0 for package metadata, checksum, owner, license, compatibility matrix, and audit reporting.
- Package Verification v0 for generated manifest, target adapters, zip archive safety, archive checksum, and registry parity.
- Install Simulation v0 for local extraction, entrypoint loading, interface loading, report presence, and adapter readability.
- Upgrade Check v0 for registry package diffs, semver bump recommendations, breaking-change notes, and release evidence.
- Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, and iteration candidate generation.
- Review Waivers v0 for human warning acceptance with reviewer, reason, scope, expiry, and blocker-safe policy.
- Governed Permission Gates v0 for reviewer-approved network, file-write, and subprocess capabilities with scope, reason, expiry, evidence, and target-enforcement mapping.
- Runtime Permission Probes v0 for packaged target adapter checks, explicit native-enforcement flags, metadata fallback evidence, and residual permission risks.
- Atlas Scope Policy v0 so examples, evolution snapshots, embedded generated skills, and validator fixtures remain visible in the full portfolio report without polluting release-actionable gates.
- Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision.
- Review Studio now reaches a clean `ready` decision for the root Meta Skill: `13 / 13` gates pass, world-class score `100`, no blockers, no warnings, and no review actions required.
This is still not the final world-class state. Registry, full target-specific compiler transforms, telemetry, and stricter governed trust gates remain open.
This is still not the final world-class state. Target-native behavior contracts are now explicit and local output-eval command execution is wired, but deeper provider-native execution transforms, real client telemetry capture, installer integration, provider-backed model-executed output eval, real human adjudication decisions, and native runtime permission enforcement remain open.
## Coverage Matrix
| 2.0 Area | Current Evidence | Status |
| --- | --- | --- |
| Skill IR | `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py` | v0 landed |
| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `reports/output_quality_scorecard.md` | v0 landed |
| Target Compiler | `scripts/compile_skill.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed |
| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_adjudication.md` | v0 landed |
| Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed |
| Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed |
| Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed |
| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` | v0 landed |
| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed |
| Review Studio Source Refs | `scripts/render_review_studio.py`, `tests/verify_review_studio.py`, `references/review-studio-method.md` | v0 landed |
| Review Annotations | `scripts/render_review_annotations.py`, `reports/review_annotations.md`, `tests/verify_review_annotations.py`, Review Studio annotation panel and blocker decision hook | v0 landed |
| Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `reports/skill_atlas.html` | v0 landed |
| Registry & Distribution | IR-sourced packager, no registry audit/package schema yet | partial |
| Telemetry & Drift | Regression history exists, no adoption or activation telemetry yet | partial |
| Compiler from IR | Packager consumes Skill IR for core semantic adapter fields and keeps frontmatter/interface parity checks | v0 landed |
| Registry & Distribution | `registry/*.schema.json`, `scripts/registry_audit.py`, `reports/registry_audit.md`, `registry/packages/yao-meta-skill.json` | v0 landed |
| Package Verification | `scripts/verify_package.py`, `reports/package_verification.md`, `tests/verify_package_verification.py` | v0 landed |
| Install Simulation | `scripts/simulate_install.py`, `reports/install_simulation.md`, `tests/verify_install_simulation.py` | v0 landed |
| Upgrade Check | `scripts/upgrade_check.py`, `reports/upgrade_check.md`, `tests/verify_upgrade_check.py` | v0 landed |
| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `reports/adoption_drift_report.md`, `tests/verify_adoption_drift.py`, `references/telemetry-drift-method.md` | v0 landed |
| Review Waivers | `scripts/render_review_waivers.py`, `reports/review_waivers.md`, `tests/verify_review_waivers.py`, `references/review-waiver-method.md` | v0 landed |
| Governed Permission Gates | `security/permission_policy.json`, `scripts/trust_check.py`, `scripts/render_review_studio.py`, `tests/verify_trust_check.py`, `tests/verify_review_studio.py` | v0 landed |
| Runtime Permission Probes | `scripts/probe_runtime_permissions.py`, `reports/runtime_permission_probes.md`, `tests/verify_runtime_permission_probes.py`, `tests/verify_review_studio.py` | v0 landed |
| Atlas Scope Policy | `skill_atlas/policy.json`, `scripts/build_skill_atlas.py`, `tests/verify_skill_atlas.py` | v0 landed |
| Compiler from IR | Packager consumes compiled target contracts for compiler provenance, generated files, adapter modes, permissions, preserved semantics, warnings, and unsupported features | v0 landed |
| Target Native Contracts | `scripts/compile_skill.py`, `scripts/cross_packager.py`, `reports/compiled_targets.md`, adapter snapshots, and generated target outputs carry native surface, activation policy, resource/script strategy, permission enforcement, install scope, review artifacts, fallback behavior, and unsupported native features | v0 landed |
## Top Findings
### 1. Compiler path is IR-first v0, but transforms are still shallow
### 1. Target-native behavior contracts are landed, but provider execution is still shallow
The packager now reads Skill IR for core semantic fields and emits provenance, schema version, job-to-be-done, semantic contract counts, governance, risk, targets, and semantic parity in each adapter.
`scripts/compile_skill.py` now reads Skill IR and emits target-specific contracts for OpenAI, Claude, generic, and Agent Skills compatible outputs. The packager embeds `compiler`, `compiled_contract`, `permission_contract`, `target_permission_contract`, `target_native_contract`, `target_transform`, warnings, and unsupported-feature notes in each adapter. The native contract makes the target surface, activation policy, resource strategy, script strategy, permission enforcement mode, install scope, review artifacts, fallback behavior, and unsupported native features reviewable instead of implicit.
Next move: add `scripts/compile_skill.py` or split the packager into target-specific transforms so OpenAI, Claude, Agent Skills, VS Code/Copilot, and generic packages can preserve runtime semantics beyond metadata.
Next move: deepen provider-native execution transforms so OpenAI, Claude, VS Code/Copilot, and generic packages can express and verify behavior through real client or installer capabilities, not only auditable contracts and fallback notes.
### 2. Output eval now meets the governed v0 minimum, but is still static
### 2. Output eval now has command-runner evidence, but provider model execution is still open
The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. The next gap is that they are still static text comparisons rather than model-executed holdout runs with timing, tokens, and reviewer adjudication.
The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. Each run emits assertion grading, execution provenance, a blind A/B review pack that hides baseline vs with-skill labels, a separate answer key, and an adjudication report from reviewer choices. The current root execution report now runs all `10` variants through `scripts/local_output_eval_runner.py`, so timing and command-runner behavior are observed instead of relying only on static recorded fixtures. It still intentionally reports `0` provider-backed model-executed runs and `10` estimated token counts, because the local runner verifies the harness contract without pretending to be a model provider. The current root adjudication report intentionally records `0 / 5` human judgments pending instead of pretending human review happened. The next gap is provider-backed model-executed holdout runs with real provider timing/tokens and real reviewer decision records.
Next move: add model-executed output eval runs, blind A/B comparison, and one real multi-file fixture.
Next move: add provider-backed model execution for holdout cases and one real multi-file fixture with actual reviewer decisions.
### 3. Review Studio is unified, but needs reviewer actions
### 3. Review Studio is unified and now has reviewer actions plus annotations
The new Review Studio page aggregates intent, trigger, output, context, runtime, trust, atlas, and release gates. It now exposes current warnings directly: low generic intent-confidence context, trust-script warnings, and portfolio-level Atlas gaps.
The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. After adding root intent context, Atlas scope policy, metadata-only telemetry evidence, runtime permission probe evidence, action source refs, and an empty annotation ledger, the current root report is clean: decision `ready`, score `100`, `13` gates, `0` gate blockers, `0` gate warnings, `0` actions, and `0` open annotation blockers.
Next move: add reviewer annotations, waived-risk records, and links from each warning to the exact source fix.
Next move: add richer source-line anchors inside generated reports and record real reviewer annotations during the next human review pass.
### 4. Multi-skill operation now has v0 coverage, but no telemetry
### 4. Multi-skill operation now has v0 coverage, but needs real telemetry
The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, and no-route opportunities. It is still static analysis and does not yet include adoption or activation telemetry.
The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, and feed next iteration candidates into Review Studio. It still needs real client-side capture instead of manual CLI event recording.
Next move: connect telemetry and failure history so Atlas can rank stale or conflicting skills by real usage impact.
Next move: connect client activations and failure history so Atlas can rank stale or conflicting skills by real usage impact.
### 5. Trust report is structural, not full security review
Trust v0 blocks obvious secrets and remote inline execution, but it does not yet execute script `--help`, inspect package archive hashes, or enforce per-target permissions.
Trust v0 blocks obvious secrets and remote inline execution, and now makes checksum scope explicit: `package_sha256` is a stable source-contract digest that excludes generated reports, packages, and raw local telemetry; archive integrity is carried by Package Verification and Registry as `archive_sha256`. Registry v0 carries both checksums into distributable metadata. Package Verification v0 now checks generated manifests, target adapters, archive path safety, required source entries, and registry parity. Install Simulation v0 now extracts the generated archive into a temporary local skill root and verifies entrypoint, manifest, interface, reports, and adapters. Upgrade Check v0 now compares package baselines, recommends semver bumps, and blocks breaking changes without an adequate declared version. Trust Report now distinguishes CLI scripts from declared internal modules, so help-surface warnings focus on executable entrypoints. Network-capable scripts are now covered by `security/network_policy.json`, with `allowed_hosts` checked against HTTPS URL literals. Trust Report also executes `python3 scripts/name.py --help` for CLI scripts with `argparse`, recording pass/fail smoke evidence without executing scripts that lack a help surface. Target adapters now carry `permission_contract` and `target_permission_contract` fields, including network, file-write, subprocess, and interactive capability counts. Governed permission gates now require reviewer-approved scope, reason, expiry, evidence, and target-enforcement mapping in `security/permission_policy.json`. Runtime Permission Probes now verify that packaged adapters expose the target permission contract and explicitly report native-enforcement limits. The remaining trust gap is native runtime permission enforcement in real clients/installers.
Next move: add stricter governed-mode gates and package hash verification after registry format lands.
Next move: add real client or installer permission enforcement integration.
## Current Gate Evidence
| Gate | Current Result |
| --- | --- |
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, and boundary coverage |
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs, `10` estimated token counts, `5` blind A/B review pairs, and `0 / 5` reviewer decisions pending |
| Runtime Conformance | `5 / 5` targets passing |
| Trust | `0` secret findings, `1` pinned dependency file, `2` network-capable scripts flagged as warnings |
| Skill Atlas | local workspace scan generated catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview |
| Review Studio | `8` gates, `0` blockers, `3` warnings, world-class score `87/100` |
| IR-first Packaging | `openai`, `claude`, and `generic` adapters include IR provenance and semantic parity checks |
| Context Budget | initial load `910/1000`, under the production budget |
| CI | `make ci-test` passed after adding v0 gates |
| Target Compiler | compiled target contracts generated for OpenAI, Claude, generic, and Agent Skills compatible outputs, including target permission contracts and target-native behavior contracts |
| Trust | `0` secret findings, `1` pinned dependency file, `3` declared internal modules, `2 / 2` network-capable scripts covered by bounded host policy, `59 / 59` CLI help smoke checks passing across `62` scripts, source-contract hash scope explicit |
| Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired |
| Runtime Permission Probes | `3 / 3` target adapters probed, `0` native-enforcement adapters, `3` explicit metadata fallbacks, `3` residual risks retained for reviewer visibility |
| Skill Atlas | `12` scanned skills, `1` actionable root skill, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `24` scoped non-actionable issue signals retained for visibility |
| Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix |
| Package Verification | `3 / 3` target adapters present, archive verified, `483` zip entries, `0` failures, `0` warnings |
| Install Simulation | archive with `483` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `3` adapters readable, `0` failures, `0` warnings |
| Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes |
| Adoption Drift | `1` metadata-only activation event, adoption `100`, risk band `low`; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages |
| Review Waivers | ledger generated; current release has no warning gates to waive, blockers remain non-waivable in v0 |
| Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers |
| Review Studio | decision `ready`, world-class score `100`, `13` gates, `0` blockers, `0` warnings, `0` review actions, `0` open annotation blockers |
| IR-first Packaging | `openai`, `claude`, and `generic` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, and semantic parity checks |
| Context Budget | initial load `987/1000`, under the production budget |
| CI | `make ci-test` target count is now `56` after adding Review Annotations |
## Next Highest-Leverage Moves
1. Split IR-first packaging into target-specific compiler transforms.
2. Add reviewer annotation and waived-risk records to Review Studio.
3. Expand Output Eval Lab from static cases to model-executed holdout and blind A/B cases.
4. Add registry package schema, package hash, and upgrade audit.
5. Connect Skill Atlas with telemetry and drift history.
1. Deepen target-native behavior contracts into provider-native execution and installer integrations.
2. Add native client or installer enforcement for approved high-permission capabilities.
3. Expand Output Eval Lab from local command-runner evidence to provider-backed model holdout and real reviewer-adjudicated cases.
4. Add real reviewer annotation records during the next human review pass.
5. Connect Skill Atlas with real client telemetry and drift history.
reports/skill-overview.html
+34 -11
View File
@@ -342,6 +342,9 @@
gap: 16px;
align-items: stretch;
}
.quality-panels {
margin-top: 16px;
}
.metrics-stack {
display: grid;
gap: 18px;
@@ -606,7 +609,7 @@
<ul class="list"><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">Skill 作者、复用团队和后续 reviewer。</span><span data-lang="en">Skill 作者、复用团队和后续 reviewer。</span></li><li><span data-lang="zh-CN">创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。</span><span data-lang="en">创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。</span></li></ul>
</aside>
</div>
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 346 个词/字，references 约 11313 个词/字。</span><span data-lang="en">入口约 346 个词/字，references 约 11313 个词/字。</span></small></article></div>
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 365 个词/字，references 约 14247 个词/字。</span><span data-lang="en">入口约 365 个词/字，references 约 14247 个词/字。</span></small></article></div>
</section>
<section id="overview">
@@ -618,7 +621,7 @@
<div class="two-col">
<article class="panel">
<h3><span data-lang="zh-CN">作用定位</span><span data-lang="en">Role</span></h3>
<ul class="list"><li><span data-lang="zh-CN">从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。</span><span data-lang="en">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</span></li><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html</span><span data-lang="en">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html</span></li></ul>
<ul class="list"><li><span data-lang="zh-CN">从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。</span><span data-lang="en">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</span></li><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html</span><span data-lang="en">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html</span></li></ul>
</article>
<figure class="chart-figure" data-chart="flow"><svg viewBox="0 0 620 170" role="img" aria-label="交付流程"><text x="20" y="28" class="chart-title">交付流程</text><path d="M188 93 H248 M398 93 H458" class="chart-line"/><g><rect x="38" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="113" y="99" text-anchor="middle">输入材料</text></g><g><rect x="248" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="323" y="99" text-anchor="middle">Skill 包体</text></g><g><rect x="458" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="533" y="99" text-anchor="middle">可复用能力</text></g></svg><figcaption>交付流程把用户输入、生成的包体和可复用能力放在一条线上。</figcaption></figure>
</div>
@@ -638,10 +641,10 @@
<article class="panel metrics-note">
<h3><span data-lang="zh-CN">指标判读</span><span data-lang="en">Reading</span></h3>
<p><span data-lang="zh-CN">先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。</span><span data-lang="en">Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.</span></p>
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 346 个词/字。</span><span data-lang="en">SKILL.md 约 346 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 346 个词/字，references 约 11313 个词/字。</span><span data-lang="en">入口约 346 个词/字，references 约 11313 个词/字。</span></small></li></ol>
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 365 个词/字。</span><span data-lang="en">SKILL.md 约 365 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 365 个词/字，references 约 14247 个词/字。</span><span data-lang="en">入口约 365 个词/字，references 约 14247 个词/字。</span></small></li></ol>
</article>
</div>
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在，便于人工阅读。</span><span data-lang="en">README.md 已存在，便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在，便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在，便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">intent-dialogue.json 已存在。</span><span data-lang="en">intent-dialogue.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 346 个词/字。</span><span data-lang="en">SKILL.md 约 346 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制，可维护性较好。</span><span data-lang="en">入口文件保持克制，可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 346 个词/字，references 约 11313 个词/字。</span><span data-lang="en">入口约 346 个词/字，references 约 11313 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高，建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高，建议压缩入口或拆分 references。</span></li></ul></div></article></div>
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在，便于人工阅读。</span><span data-lang="en">README.md 已存在，便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在，便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在，便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">compiled_targets.json 已存在。</span><span data-lang="en">compiled_targets.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 365 个词/字。</span><span data-lang="en">SKILL.md 约 365 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制，可维护性较好。</span><span data-lang="en">入口文件保持克制，可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 365 个词/字，references 约 14247 个词/字。</span><span data-lang="en">入口约 365 个词/字，references 约 14247 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高，建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高，建议压缩入口或拆分 references。</span></li></ul></div></article></div>
</div>
</div>
</div>
@@ -654,10 +657,10 @@
<p><span data-lang="zh-CN">判断这个 Skill 在能力地图中的位置和复用范围。</span><span data-lang="en">Places this skill on a capability map and clarifies reuse scope.</span></p>
</div>
<div class="two-col">
<figure class="chart-figure" data-chart="matrix"><svg viewBox="0 0 520 460" role="img" aria-label="能力矩阵"><text x="20" y="30" class="chart-title">能力矩阵</text><rect x="70" y="70" width="380" height="320" fill="#faf9f5" stroke="#e8e6dc"/><line x1="260" y1="70" x2="260" y2="390" stroke="#e8e6dc"/><line x1="70" y1="230" x2="450" y2="230" stroke="#e8e6dc"/><text x="260" y="424" text-anchor="middle">执行确定性</text><text x="22" y="230" transform="rotate(-90 22 230)" text-anchor="middle">知识密度</text><circle cx="343.6" cy="174.0" r="12" fill="#1B365D"/><text x="361.6" y="179.0">Dialogue interaction</text></svg><figcaption>能力矩阵说明这个 Skill 更偏知识密集还是执行确定。</figcaption></figure>
<figure class="chart-figure" data-chart="matrix"><svg viewBox="0 0 520 460" role="img" aria-label="能力矩阵"><text x="20" y="30" class="chart-title">能力矩阵</text><rect x="70" y="70" width="380" height="320" fill="#faf9f5" stroke="#e8e6dc"/><line x1="260" y1="70" x2="260" y2="390" stroke="#e8e6dc"/><line x1="70" y1="230" x2="450" y2="230" stroke="#e8e6dc"/><text x="260" y="424" text-anchor="middle">执行确定性</text><text x="22" y="230" transform="rotate(-90 22 230)" text-anchor="middle">知识密度</text><circle cx="343.6" cy="174.0" r="12" fill="#1B365D"/><text x="361.6" y="179.0">Execution operation</text></svg><figcaption>能力矩阵说明这个 Skill 更偏知识密集还是执行确定。</figcaption></figure>
<article class="panel">
<h3><span data-lang="zh-CN">画像摘要</span><span data-lang="en">Profile</span></h3>
<ul class="list"><li><span data-lang="zh-CN">能力类型：Dialogue interaction</span><span data-lang="en">能力类型：Dialogue interaction</span></li><li><span data-lang="zh-CN">成熟度：governed</span><span data-lang="en">成熟度：governed</span></li><li><span data-lang="zh-CN">触发强度：手动触发 + description 路由</span><span data-lang="en">触发强度：手动触发 + description 路由</span></li><li><span data-lang="zh-CN">复用范围：跨平台</span><span data-lang="en">复用范围：跨平台</span></li></ul>
<ul class="list"><li><span data-lang="zh-CN">能力类型：Execution operation</span><span data-lang="en">能力类型：Execution operation</span></li><li><span data-lang="zh-CN">成熟度：governed</span><span data-lang="en">成熟度：governed</span></li><li><span data-lang="zh-CN">触发强度：手动触发 + description 路由</span><span data-lang="en">触发强度：手动触发 + description 路由</span></li><li><span data-lang="zh-CN">复用范围：跨平台</span><span data-lang="en">复用范围：跨平台</span></li></ul>
</article>
</div>
</div>
@@ -696,11 +699,11 @@
<h3><span data-lang="zh-CN">触发描述</span><span data-lang="en">Trigger</span></h3>
<p><span data-lang="zh-CN">该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。</span><span data-lang="en">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</span></p>
<h3><span data-lang="zh-CN">输入材料</span><span data-lang="en">Inputs</span></h3>
<ul class="list"><li><span data-lang="zh-CN">用户提供的工作流、提示词、文档、记录或散乱笔记</span><span data-lang="en">用户提供的工作流、提示词、文档、记录或散乱笔记</span></li><li><span data-lang="zh-CN">期望沉淀的复用场景、排除项、约束和质量标准</span><span data-lang="en">期望沉淀的复用场景、排除项、约束和质量标准</span></li></ul>
<ul class="list"><li><span data-lang="zh-CN">原始说明可切换到英文查看；默认中文报告保留结论与结构说明。</span><span data-lang="en">rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions</span></li><li><span data-lang="zh-CN">原始说明可切换到英文查看；默认中文报告保留结论与结构说明。</span><span data-lang="en">an existing skill directory that needs refactor, evaluation, packaging, or governance hardening</span></li><li><span data-lang="zh-CN">原始说明可切换到英文查看；默认中文报告保留结论与结构说明。</span><span data-lang="en">target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution</span></li><li><span data-lang="zh-CN">原始说明可切换到英文查看；默认中文报告保留结论与结构说明。</span><span data-lang="en">benchmark references, local constraints, desired maturity tier, and review standards</span></li></ul>
</article>
<article class="panel">
<h3><span data-lang="zh-CN">输出结果</span><span data-lang="en">Outputs</span></h3>
<ul class="list"><li><span data-lang="zh-CN">可路由的 SKILL.md</span><span data-lang="en">可路由的 SKILL.md</span></li><li><span data-lang="zh-CN">agents/interface.yaml 元数据</span><span data-lang="en">agents/interface.yaml 元数据</span></li><li><span data-lang="zh-CN">必要的 references、scripts、evals、reports 证据</span><span data-lang="en">必要的 references、scripts、evals、reports 证据</span></li><li><span data-lang="zh-CN">结构化 Skill 目录，共 8 类关键资产。</span><span data-lang="en">结构化 Skill 目录，共 8 类关键资产。</span></li></ul>
<ul class="list"><li><span data-lang="zh-CN">原始说明可切换到英文查看；默认中文报告保留结论与结构说明。</span><span data-lang="en">A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.</span></li><li><span data-lang="zh-CN">结构化 Skill 目录，共 8 类关键资产。</span><span data-lang="en">结构化 Skill 目录，共 8 类关键资产。</span></li></ul>
<h3><span data-lang="zh-CN">不应触发</span><span data-lang="en">Should Not Trigger</span></h3>
<ul class="list"><li><span data-lang="zh-CN">只需要一次性回答、没有复用价值的临时请求。</span><span data-lang="en">只需要一次性回答、没有复用价值的临时请求。</span></li><li><span data-lang="zh-CN">要求直接执行相邻任务，而不是沉淀或使用这个 Skill。</span><span data-lang="en">要求直接执行相邻任务，而不是沉淀或使用这个 Skill。</span></li><li><span data-lang="zh-CN">缺少必要事实且用户不允许澄清的场景。</span><span data-lang="en">缺少必要事实且用户不允许澄清的场景。</span></li></ul>
</article>
@@ -717,8 +720,28 @@
<div>
<table>
<thead><tr><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th><th><span data-lang="zh-CN">证据</span><span data-lang="en">Evidence</span></th><th><span data-lang="zh-CN">建议</span><span data-lang="en">Action</span></th></tr></thead>
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简，并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。</span><span data-lang="en">已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。</span><span data-lang="en">上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简，并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span><span data-lang="en">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。</span><span data-lang="en">上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
</table>
<div class="two-col quality-panels">
<article class="panel">
<h3><span data-lang="zh-CN">执行证据</span><span data-lang="en">Execution Evidence</span></h3>
<ul class="list"><li><span data-lang="zh-CN">变体运行：10</span><span data-lang="en">变体运行：10</span></li><li><span data-lang="zh-CN">模型执行：0</span><span data-lang="en">模型执行：0</span></li><li><span data-lang="zh-CN">记录样本：0</span><span data-lang="en">记录样本：0</span></li><li><span data-lang="zh-CN">Token 估算：10</span><span data-lang="en">Token 估算：10</span></li></ul>
</article>
<article class="panel">
<h3><span data-lang="zh-CN">盲评审定</span><span data-lang="en">Blind Adjudication</span></h3>
<ul class="list"><li><span data-lang="zh-CN">评审进度：0 / 5</span><span data-lang="en">评审进度：0 / 5</span></li><li><span data-lang="zh-CN">待评审：5</span><span data-lang="en">待评审：5</span></li><li><span data-lang="zh-CN">一致率：暂无</span><span data-lang="en">一致率：暂无</span></li><li><span data-lang="zh-CN">非法决策：0</span><span data-lang="en">非法决策：0</span></li></ul>
</article>
</div>
<div class="two-col quality-panels">
<article class="panel">
<h3><span data-lang="zh-CN">评审原则</span><span data-lang="en">Review Rule</span></h3>
<ul class="list"><li><span data-lang="zh-CN">先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。</span><span data-lang="en">Record the reviewer&#x27;s A/B choice before opening the answer key and calculating agreement.</span></li><li><span data-lang="zh-CN">缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。</span><span data-lang="en">When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.</span></li></ul>
</article>
<article class="panel">
<h3><span data-lang="zh-CN">运行原则</span><span data-lang="en">Run Rule</span></h3>
<ul class="list"><li><span data-lang="zh-CN">recorded fixture 只能证明可复现样本，不等同于模型执行。</span><span data-lang="en">A recorded fixture proves reproducible samples only; it is not model execution.</span></li><li><span data-lang="zh-CN">只有 provider runner 返回 model metadata 时才计入 model-executed。</span><span data-lang="en">Only provider runners that return model metadata count as model-executed.</span></li></ul>
</article>
</div>
</div>
</div>
</section>
@@ -734,7 +757,7 @@
<div>
<table>
<thead><tr><th><span data-lang="zh-CN">风险</span><span data-lang="en">Risk</span></th><th><span data-lang="zh-CN">信号</span><span data-lang="en">Signal</span></th><th><span data-lang="zh-CN">应对</span><span data-lang="en">Response</span></th></tr></thead>
<tbody><tr><td><span data-lang="zh-CN">误触发风险</span><span data-lang="en">误触发风险</span></td><td><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">输出漂移风险</span><span data-lang="en">输出漂移风险</span></td><td><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">证据不足风险</span><span data-lang="en">证据不足风险</span></td><td><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">包体膨胀风险</span><span data-lang="en">包体膨胀风险</span></td><td><span data-lang="zh-CN">SKILL.md 约 346 个词/字。</span><span data-lang="en">SKILL.md 约 346 个词/字。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">跨平台迁移风险</span><span data-lang="en">跨平台迁移风险</span></td><td><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr></tbody>
<tbody><tr><td><span data-lang="zh-CN">误触发风险</span><span data-lang="en">误触发风险</span></td><td><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">输出漂移风险</span><span data-lang="en">输出漂移风险</span></td><td><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">证据不足风险</span><span data-lang="en">证据不足风险</span></td><td><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">包体膨胀风险</span><span data-lang="en">包体膨胀风险</span></td><td><span data-lang="zh-CN">SKILL.md 约 365 个词/字。</span><span data-lang="en">SKILL.md 约 365 个词/字。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">跨平台迁移风险</span><span data-lang="en">跨平台迁移风险</span></td><td><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></td><td><span data-lang="zh-CN">先补证据和边界，再增加包体复杂度。</span><span data-lang="en">先补证据和边界，再增加包体复杂度。</span></td></tr></tbody>
</table>
</div>
</div>
@@ -748,7 +771,7 @@
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
</div>
<div class="two-col">
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-1.2" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-2.4" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-3.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="34.9 65.1" stroke-dashoffset="-4.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="60.2 39.8" stroke-dashoffset="-39.8" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">255项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="1.0 99.0" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="1.0 99.0" stroke-dashoffset="-1.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="1.0 99.0" stroke-dashoffset="-2.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="1.0 99.0" stroke-dashoffset="-3.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="32.7 67.3" stroke-dashoffset="-4.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="63.3 36.7" stroke-dashoffset="-36.7" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">296项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
<table>
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
reports/skill-overview.json
+537 -71
View File
@@ -12,10 +12,23 @@
"SKILL.md",
"agents/interface.yaml",
"reports/skill-ir.json",
"reports/compiled_targets.md",
"reports/output_quality_scorecard.md",
"reports/output_execution_runs.md",
"reports/output_blind_review_pack.md",
"reports/output_blind_answer_key.json",
"reports/output_review_adjudication.md",
"reports/conformance_matrix.md",
"reports/security_trust_report.md",
"reports/runtime_permission_probes.md",
"reports/skill_atlas.html",
"reports/registry_audit.md",
"reports/package_verification.md",
"reports/install_simulation.md",
"reports/upgrade_check.md",
"reports/adoption_drift_report.md",
"reports/review_waivers.md",
"reports/review_annotations.md",
"reports/review-studio.html",
"reports/skill-overview.html"
],
@@ -52,17 +65,17 @@
"label": "证据充分",
"score": 100,
"reasons": [
"已生成 13 / 13 类报告证据。",
"已生成 20 / 20 类报告证据。",
"skill-ir.json 已存在。",
"intent-dialogue.json 已存在。",
"intent-confidence.json 已存在。"
"compiled_targets.json 已存在。",
"intent-dialogue.json 已存在。"
]
},
"maintainability_score": {
"label": "可维护性",
"score": 100,
"reasons": [
"SKILL.md 约 346 个词/字。",
"SKILL.md 约 365 个词/字。",
"入口文件保持克制，可维护性较好。",
"references/ 已承载扩展指导。",
"scripts/ 已承载确定性逻辑。",
@@ -83,7 +96,7 @@
"label": "上下文成本",
"score": 42,
"reasons": [
"入口约 346 个词/字，references 约 11313 个词/字。",
"入口约 365 个词/字，references 约 14247 个词/字。",
"分数越高代表上下文成本越低。",
"上下文成本偏高，建议压缩入口或拆分 references。"
]
@@ -91,7 +104,7 @@
},
"capability_profile": {
"archetype": "governed",
"task_family": "Dialogue interaction",
"task_family": "Execution operation",
"maturity": "governed",
"trigger_strength": "手动触发 + description 路由",
"reuse_scope": "跨平台",
@@ -104,7 +117,7 @@
"nodes": [
{
"title": "意图澄清",
"body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
"body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse."
},
{
"title": "边界路由",
@@ -124,7 +137,7 @@
},
{
"title": "杠杆升级",
"body": "Ask one focused question and update intent context before adding assets."
"body": "Name the recurring job, expected input, output, and strongest non-goal in compact language."
}
],
"layers": [
@@ -143,13 +156,13 @@
"shell": "bash"
},
"inputs": [
"用户提供的工作流、提示词、文档、记录或散乱笔记",
"期望沉淀的复用场景、排除项、约束和质量标准"
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"outputs": [
"可路由的 SKILL.md",
"agents/interface.yaml 元数据",
"必要的 references、scripts、evals、reports 证据",
"A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"结构化 Skill 目录，共 8 类关键资产。"
],
"should_trigger": [
@@ -181,13 +194,13 @@
"strengths": [
"触发面保持精简，并锚定在 frontmatter description。",
"已生成 Skill IR，核心语义可先于平台打包被审查和迁移。",
"已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。",
"已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。",
"已生成 Runtime Conformance Matrix，可审查目标端消费能力。",
"已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。",
"已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。"
"已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。",
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。"
],
"gaps": [
"上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。"
"上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。"
],
"recommendations": [
"先改触发边界，再扩展工作流。",
@@ -195,7 +208,7 @@
"每次升级后重新生成报告并检查分数原因。"
],
"artifact_design": {
"design_system": "review studio",
"design_system": "metric editorial",
"highlights": [
"Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
"Name the working directory and required inputs for commands.",
@@ -204,24 +217,24 @@
]
},
"prompt_quality": {
"overall_quality_score": 82.0,
"overall_quality_score": 89.0,
"highlights": [
"Primary prompt task family: Dialogue interaction.",
"Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling",
"Completeness: 80/100.",
"Clarity: 80/100."
"Primary prompt task family: Execution operation.",
"Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints",
"Completeness: 100/100.",
"Clarity: 85/100."
]
},
"system_model": {
"stability": {
"score": 86,
"band": "stable-first-pass"
"score": 100,
"band": "system-ready"
},
"highlights": [
"Stability: stable-first-pass (86/100).",
"Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.",
"Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language."
"Stability: system-ready (100/100).",
"Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.",
"Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes."
]
}
},
@@ -238,21 +251,21 @@
"name": "输出漂移风险",
"impact": 2,
"probability": 1,
"signal": "已生成 13 / 13 类报告证据。",
"signal": "已生成 20 / 20 类报告证据。",
"response": "先补证据和边界，再增加包体复杂度。"
},
{
"name": "证据不足风险",
"impact": 3,
"probability": 1,
"signal": "已生成 13 / 13 类报告证据。",
"signal": "已生成 20 / 20 类报告证据。",
"response": "先补证据和边界，再增加包体复杂度。"
},
{
"name": "包体膨胀风险",
"impact": 2,
"probability": 1,
"signal": "SKILL.md 约 346 个词/字。",
"signal": "SKILL.md 约 365 个词/字。",
"response": "先补证据和边界，再增加包体复杂度。"
},
{
@@ -414,13 +427,13 @@
"path": "references",
"label": "Extended guidance and reusable notes",
"kind": "folder",
"file_count": 29
"file_count": 32
},
{
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 50
"file_count": 62
},
{
"path": "evals",
@@ -432,10 +445,10 @@
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 144
"file_count": 170
}
],
"file_count": 255,
"file_count": 296,
"folder_count": 4,
"distribution": [
{
@@ -456,11 +469,11 @@
},
{
"label": "references",
"value": 29
"value": 32
},
{
"label": "scripts",
"value": 50
"value": 62
},
{
"label": "evals",
@@ -468,7 +481,7 @@
},
{
"label": "reports",
"value": 144
"value": 170
}
]
},
@@ -583,13 +596,13 @@
"path": "references",
"label": "Extended guidance and reusable notes",
"kind": "folder",
"file_count": 29
"file_count": 32
},
{
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 50
"file_count": 62
},
{
"path": "evals",
@@ -601,16 +614,16 @@
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 144
"file_count": 170
}
],
"strengths": [
"触发面保持精简，并锚定在 frontmatter description。",
"已生成 Skill IR，核心语义可先于平台打包被审查和迁移。",
"已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。",
"已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。",
"已生成 Runtime Conformance Matrix，可审查目标端消费能力。",
"已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。",
"已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。"
"已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。",
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。"
],
"scenarios": [
"把重复流程沉淀为可复用的 agent skill。",
@@ -626,20 +639,20 @@
},
"io_contract": {
"inputs": [
"用户提供的工作流、提示词、文档、记录或散乱笔记",
"期望沉淀的复用场景、排除项、约束和质量标准"
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"outputs": [
"可路由的 SKILL.md",
"agents/interface.yaml 元数据",
"必要的 references、scripts、evals、reports 证据",
"A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"结构化 Skill 目录，共 8 类关键资产。"
]
},
"principles": [
{
"title": "意图澄清",
"body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
"body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse."
},
{
"title": "边界路由",
@@ -659,7 +672,7 @@
},
{
"title": "杠杆升级",
"body": "Ask one focused question and update intent context before adding assets."
"body": "Name the recurring job, expected input, output, and strongest non-goal in compact language."
}
],
"roadmap": [
@@ -707,6 +720,81 @@
"trigger_samples": 8,
"output_eval_cases": 3
},
"compiled_targets": {
"ok": true,
"schema_version": "1.0",
"summary": {
"target_count": 4,
"pass_count": 4,
"warn_count": 0,
"block_count": 0,
"failure_count": 0,
"warning_count": 0
},
"targets": [
{
"target": "openai",
"status": "pass",
"adapter_mode": "metadata-adapter",
"degradation_strategy": "metadata-adapter",
"native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source",
"permission_enforcement": "metadata-only",
"generated_files": [
"targets/openai/adapter.json",
"targets/openai/agents/openai.yaml"
],
"unsupported_features": [
"client-native script permission prompts are represented as permission contract metadata"
],
"warnings": []
},
{
"target": "claude",
"status": "pass",
"adapter_mode": "neutral-source-plus-adapter",
"degradation_strategy": "neutral-source-plus-adapter",
"native_surface": "Claude-compatible neutral source folder with adapter notes",
"permission_enforcement": "metadata-fallback",
"generated_files": [
"targets/claude/adapter.json",
"targets/claude/README.md"
],
"unsupported_features": [
"vendor-native metadata fields are carried as adapter JSON and README notes"
],
"warnings": []
},
{
"target": "generic",
"status": "pass",
"adapter_mode": "agent-skills-compatible",
"degradation_strategy": "neutral-source",
"native_surface": "Agent Skills compatible neutral package",
"permission_enforcement": "consumer-enforced-or-metadata-only",
"generated_files": [
"targets/generic/adapter.json"
],
"unsupported_features": [],
"warnings": []
},
{
"target": "agent-skills-compatible",
"status": "pass",
"adapter_mode": "neutral-agent-skills-source",
"degradation_strategy": "neutral-source",
"native_surface": "Agent Skills standard source tree",
"permission_enforcement": "consumer-enforced-or-metadata-only",
"generated_files": [
"SKILL.md",
"agents/interface.yaml"
],
"unsupported_features": [],
"warnings": []
}
],
"failures": [],
"warnings": []
},
"output_quality": {
"case_count": 5,
"file_backed_case_count": 1,
@@ -717,39 +805,417 @@
"delta": 100.0,
"regression_count": 0,
"gate_pass": true,
"failure_taxonomy": []
"failure_taxonomy": [],
"blind_pair_count": 5
},
"output_execution": {
"ok": true,
"summary": {
"case_count": 5,
"variant_run_count": 10,
"command_executed_count": 10,
"model_executed_count": 0,
"recorded_fixture_count": 0,
"timing_observed_count": 10,
"token_observed_count": 0,
"token_estimated_count": 10,
"baseline_pass_rate": 0.0,
"with_skill_pass_rate": 100.0,
"delta": 100.0,
"regression_count": 0,
"failure_count": 0,
"gate_pass": true
},
"runner": {
"mode": "command",
"command": [
"python3",
"scripts/local_output_eval_runner.py"
],
"timeout_seconds": 30.0
},
"failures": []
},
"output_blind_review": {
"summary": {
"pair_count": 5,
"answer_key_separate": true,
"with_skill_hidden_count": 5
},
"seed": "yao-output-eval-blind-v1",
"pair_count": 5,
"answer_key_separate": true
},
"output_review_adjudication": {
"ok": true,
"summary": {
"pair_count": 5,
"judgment_count": 0,
"pending_count": 5,
"agreement_count": 0,
"disagreement_count": 0,
"invalid_decision_count": 0,
"agreement_rate": null,
"needs_review": true,
"failure_count": 0
},
"reviewer": "",
"reviewed_at": "",
"failures": []
},
"runtime_conformance": {
"target_count": 5,
"pass_count": 5,
"fail_count": 0
},
"runtime_permissions": {
"ok": true,
"summary": {
"target_count": 3,
"pass_count": 3,
"fail_count": 0,
"native_enforcement_count": 0,
"metadata_fallback_count": 3,
"residual_risk_count": 3,
"required_capability_count": 3,
"failure_count": 0
},
"expected_capabilities": [
"file_write",
"network",
"subprocess"
],
"targets": [
{
"target": "openai",
"status": "pass",
"assurance": "metadata-fallback-explicit",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
},
{
"target": "claude",
"status": "pass",
"assurance": "metadata-fallback-explicit",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
},
{
"target": "generic",
"status": "pass",
"assurance": "metadata-fallback-explicit",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
}
],
"failures": []
},
"trust_security": {
"scanned_files": 127,
"script_count": 50,
"scanned_files": 146,
"script_count": 62,
"internal_module_count": 3,
"secret_findings": 0,
"dependency_files": [
"requirements-ci.txt"
],
"network_script_count": 2,
"network_policy_covered_count": 2,
"network_policy_missing_count": 0,
"file_write_script_count": 47,
"permission_required_count": 3,
"permission_approved_count": 3,
"permission_missing_count": 0,
"permission_invalid_count": 0,
"permission_expired_count": 0,
"help_smoke_checked_count": 59,
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad"
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 146,
"package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe"
},
"skill_atlas": {
"skill_count": 12,
"actionable_skill_count": 1,
"route_collision_count": 5,
"actionable_route_collision_count": 0,
"owner_gap_count": 9,
"actionable_owner_gap_count": 0,
"stale_count": 10,
"actionable_stale_count": 0,
"shared_resource_count": 0,
"no_route_opportunity_count": 0
"no_route_opportunity_count": 0,
"non_actionable_issue_count": 24
},
"registry_distribution": {
"ok": true,
"package": {
"schema_version": "2.0",
"name": "yao-meta-skill",
"version": "1.1.0",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"targets": [
"openai",
"claude",
"generic",
"agent-skills-compatible"
],
"maturity": "governed",
"owner": "Yao Team",
"review_cadence": "quarterly",
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe",
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06"
},
"compatibility": {
"openai": "pass",
"claude": "pass",
"agent-skills": "pass",
"vscode": "pass",
"generic": "pass",
"agent-skills-compatible": "pass"
},
"source": {
"skill_dir": ".",
"skill_ir": "skill-ir/examples/yao-meta-skill.json",
"ir_schema_version": "2.0.0",
"canonical_metadata": "agents/interface.yaml"
},
"artifacts": {
"overview_html": "reports/skill-overview.html",
"review_studio_html": "reports/review-studio.html",
"trust_report": "reports/security_trust_report.md",
"conformance_matrix": "reports/conformance_matrix.md",
"compiled_targets": "reports/compiled_targets.md",
"package_verification": "reports/package_verification.md",
"install_simulation": "reports/install_simulation.md",
"adoption_drift": "reports/adoption_drift_report.md",
"review_waivers": "reports/review_waivers.md",
"review_annotations": "reports/review_annotations.md",
"package_metadata": "registry/packages/yao-meta-skill.json"
},
"distribution": {
"archive_verified": true,
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
},
"generated_at": "2026-06-13"
},
"failures": [],
"warnings": []
},
"package_verification": {
"ok": true,
"summary": {
"target_count": 3,
"adapter_count": 3,
"archive_present": true,
"archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06",
"archive_entry_count": 483,
"failure_count": 0,
"warning_count": 0
},
"failures": [],
"warnings": []
},
"install_simulation": {
"ok": true,
"summary": {
"archive_present": true,
"archive_entry_count": 483,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
"interface_loaded": true,
"adapter_count": 3,
"install_root_is_temp": false,
"failure_count": 0,
"warning_count": 0
},
"failures": [],
"warnings": []
},
"upgrade_check": {
"ok": true,
"summary": {
"declared_bump": "minor",
"recommended_bump": "minor",
"breaking_change_count": 0,
"added_target_count": 1,
"metadata_change_count": 1,
"checksum_change_count": 2,
"failure_count": 0,
"warning_count": 0
},
"upgrade_diff": {
"name_changed": false,
"added_targets": [
"agent-skills-compatible"
],
"removed_targets": [],
"compatibility_changes": [
{
"target": "agent-skills",
"from": "missing",
"to": "pass",
"direction": "improved"
},
{
"target": "agent-skills-compatible",
"from": "missing",
"to": "pass",
"direction": "improved"
},
{
"target": "vscode",
"from": "missing",
"to": "pass",
"direction": "improved"
}
],
"metadata_changes": [
{
"field": "description",
"from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.",
"to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
}
],
"checksum_changes": [
{
"field": "archive_sha256",
"from": "",
"to": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe"
}
]
},
"release_notes": [
"Recommended version bump: minor.",
"Added targets: agent-skills-compatible.",
"Package or archive checksum changed; reviewers should verify package artifacts before release.",
"Package verification evidence: reports/package_verification.md."
],
"failures": [],
"warnings": []
},
"adoption_drift": {
"ok": true,
"summary": {
"event_count": 1,
"activation_count": 1,
"accepted_count": 1,
"edited_count": 0,
"rejected_count": 0,
"missed_count": 0,
"failed_count": 0,
"adoption_rate": 100.0,
"missed_trigger_count": 0,
"wrong_trigger_count": 0,
"bad_output_count": 0,
"script_error_count": 0,
"missing_resource_count": 0,
"review_overdue_count": 0,
"risk_band": "low",
"event_types": {
"skill_activation": 1
},
"failure_types": {}
},
"next_iteration_candidates": [],
"privacy_contract": {
"storage": "local-first",
"event_scope": "metadata-only",
"raw_content_allowed": false,
"raw_event_log_packaged": false,
"blocked_fields": [
"content",
"input",
"inputs",
"message",
"messages",
"note",
"output",
"outputs",
"prompt",
"raw",
"text",
"transcript"
]
},
"failures": []
},
"review_waivers": {
"ok": true,
"summary": {
"waiver_count": 0,
"active_count": 0,
"expired_count": 0,
"invalid_count": 0,
"covered_gate_count": 0,
"covered_gate_keys": []
},
"policy": {
"blocker_waivers_allowed": false,
"minimum_reason_chars": 20,
"expires_required": true,
"known_gate_keys": [
"context-budget",
"intent-canvas",
"operations-loop",
"output-lab",
"permission-gates",
"permission-runtime",
"registry-audit",
"release-notes",
"runtime-matrix",
"skill-atlas",
"trigger-lab",
"trust-report"
]
},
"failures": [],
"warnings": []
},
"review_annotations": {
"ok": true,
"summary": {
"annotation_count": 0,
"open_count": 0,
"resolved_count": 0,
"deferred_count": 0,
"open_blocker_count": 0,
"open_warning_count": 0,
"linked_gate_count": 0,
"target_missing_count": 0,
"failure_count": 0
},
"annotations": [],
"failures": []
},
"synthesis_highlights": [
"Borrow a first-time operator flow that explains itself before it asks for more structure.",
"Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.",
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed.",
"Borrow the discipline of defining what the skill should not own before growing the package."
],
"artifact_design": {
"design_system": "review studio",
"design_system": "metric editorial",
"highlights": [
"Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
"Name the working directory and required inputs for commands.",
@@ -758,24 +1224,24 @@
]
},
"prompt_quality": {
"overall_quality_score": 82.0,
"overall_quality_score": 89.0,
"highlights": [
"Primary prompt task family: Dialogue interaction.",
"Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling",
"Completeness: 80/100.",
"Clarity: 80/100."
"Primary prompt task family: Execution operation.",
"Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints",
"Completeness: 100/100.",
"Clarity: 85/100."
]
},
"system_model": {
"stability": {
"score": 86,
"band": "stable-first-pass"
"score": 100,
"band": "system-ready"
},
"highlights": [
"Stability: stable-first-pass (86/100).",
"Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.",
"Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language."
"Stability: system-ready (100/100).",
"Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.",
"Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes."
]
},
"metadata": {
reports/skill_atlas.html
+12 -7
View File
@@ -25,20 +25,25 @@
<p>Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.</p>
<section class="grid">
<div class="card"><span>Skills</span><strong>12</strong></div>
<div class="card"><span>Route Collisions</span><strong>5</strong></div>
<div class="card"><span>Owner Gaps</span><strong>9</strong></div>
<div class="card"><span>Stale Skills</span><strong>10</strong></div>
<div class="card"><span>Actionable</span><strong>1</strong></div>
<div class="card"><span>Route Collisions</span><strong>0</strong></div>
<div class="card"><span>Owner Gaps</span><strong>0</strong></div>
<div class="card"><span>Stale Skills</span><strong>0</strong></div>
<div class="card"><span>No-Route Opportunities</span><strong>0</strong></div>
</section>
<section>
<h2>Top Issues</h2>
<ul><li><strong>frontend-review</strong> collision</li><li><strong>frontend-review</strong> collision</li><li><strong>frontend-review</strong> collision</li><li><strong>broken-yaml-skill</strong> collision</li><li><strong>frontend-review</strong> duplicate-name</li><li><strong>frontend-review</strong> owner, review_cadence, maturity</li><li><strong>frontend-review</strong> owner, review_cadence, maturity</li><li><strong>frontend-review</strong> owner, review_cadence, maturity</li><li><strong>note-cleanup</strong> owner, review_cadence, maturity</li><li><strong>frontend-review</strong> owner, review_cadence, maturity</li><li><strong>geo-ranking-article-generator</strong> owner, review_cadence, maturity</li><li><strong>invalid-governance-skill</strong> owner, review_cadence</li><li><strong>broken-yaml-skill</strong> owner, review_cadence, maturity</li><li><strong>broken-skill</strong> owner, review_cadence, maturity</li><li><strong>frontend-review</strong> missing updated_at</li><li><strong>frontend-review</strong> missing updated_at</li><li><strong>frontend-review</strong> missing updated_at</li><li><strong>incident-command-governor</strong> review overdue by cadence monthly</li><li><strong>note-cleanup</strong> missing updated_at</li><li><strong>frontend-review</strong> missing updated_at</li><li><strong>geo-ranking-article-generator</strong> missing updated_at</li><li><strong>invalid-governance-skill</strong> missing updated_at</li><li><strong>broken-yaml-skill</strong> missing updated_at</li><li><strong>broken-skill</strong> missing updated_at</li></ul>
<h2>Actionable Issues</h2>
<ul><li>No blocking portfolio issues detected.</li></ul>
</section>
<section>
<h2>Full Portfolio Counts</h2>
<p>All scanned skills remain visible: 5 total route collisions, 9 total owner gaps, and 10 total stale signals.</p>
</section>
<section>
<h2>Catalog</h2>
<table>
<thead><tr><th>Name</th><th>Path</th><th>Owner</th><th>Maturity</th><th>Review</th></tr></thead>
<tbody><tr><td>yao-meta-skill</td><td>.</td><td>Yao Team</td><td>governed</td><td>quarterly</td></tr><tr><td>release-orchestrator</td><td>examples/complex-release-orchestrator/generated-skill</td><td>Yao Team</td><td>production</td><td>quarterly</td></tr><tr><td>frontend-review</td><td>examples/evolution-frontend-review/final</td><td>missing</td><td>unknown</td><td>missing</td></tr><tr><td>frontend-review</td><td>examples/evolution-frontend-review/v0</td><td>missing</td><td>unknown</td><td>missing</td></tr><tr><td>frontend-review</td><td>examples/evolution-frontend-review/v1</td><td>missing</td><td>unknown</td><td>missing</td></tr><tr><td>incident-command-governor</td><td>examples/governed-incident-command/generated-skill</td><td>Incident Operations Team</td><td>governed</td><td>monthly</td></tr><tr><td>note-cleanup</td><td>examples/simple-note-cleanup/generated-skill</td><td>missing</td><td>unknown</td><td>missing</td></tr><tr><td>frontend-review</td><td>examples/team-frontend-review/generated-skill</td><td>missing</td><td>unknown</td><td>missing</td></tr><tr><td>geo-ranking-article-generator</td><td>geo-ranking-article-generator</td><td>missing</td><td>unknown</td><td>missing</td></tr><tr><td>invalid-governance-skill</td><td>tests/fixtures/governance_invalid_manifest</td><td>missing</td><td>governed</td><td>missing</td></tr><tr><td>broken-yaml-skill</td><td>tests/fixtures/package_invalid_yaml</td><td>missing</td><td>unknown</td><td>missing</td></tr><tr><td>broken-skill</td><td>tests/fixtures/package_missing_interface_field</td><td>missing</td><td>unknown</td><td>missing</td></tr></tbody>
<thead><tr><th>Name</th><th>Path</th><th>Owner</th><th>Maturity</th><th>Review</th><th>Scope</th></tr></thead>
<tbody><tr><td>yao-meta-skill</td><td>.</td><td>Yao Team</td><td>governed</td><td>quarterly</td><td>release</td></tr><tr><td>release-orchestrator</td><td>examples/complex-release-orchestrator/generated-skill</td><td>Yao Team</td><td>production</td><td>quarterly</td><td>example</td></tr><tr><td>frontend-review</td><td>examples/evolution-frontend-review/final</td><td>missing</td><td>unknown</td><td>missing</td><td>example</td></tr><tr><td>frontend-review</td><td>examples/evolution-frontend-review/v0</td><td>missing</td><td>unknown</td><td>missing</td><td>example</td></tr><tr><td>frontend-review</td><td>examples/evolution-frontend-review/v1</td><td>missing</td><td>unknown</td><td>missing</td><td>example</td></tr><tr><td>incident-command-governor</td><td>examples/governed-incident-command/generated-skill</td><td>Incident Operations Team</td><td>governed</td><td>monthly</td><td>example</td></tr><tr><td>note-cleanup</td><td>examples/simple-note-cleanup/generated-skill</td><td>missing</td><td>unknown</td><td>missing</td><td>example</td></tr><tr><td>frontend-review</td><td>examples/team-frontend-review/generated-skill</td><td>missing</td><td>unknown</td><td>missing</td><td>example</td></tr><tr><td>geo-ranking-article-generator</td><td>geo-ranking-article-generator</td><td>missing</td><td>unknown</td><td>missing</td><td>embedded-generated-skill</td></tr><tr><td>invalid-governance-skill</td><td>tests/fixtures/governance_invalid_manifest</td><td>missing</td><td>governed</td><td>missing</td><td>test-fixture</td></tr><tr><td>broken-yaml-skill</td><td>tests/fixtures/package_invalid_yaml</td><td>missing</td><td>unknown</td><td>missing</td><td>test-fixture</td></tr><tr><td>broken-skill</td><td>tests/fixtures/package_missing_interface_field</td><td>missing</td><td>unknown</td><td>missing</td><td>test-fixture</td></tr></tbody>
</table>
</section>
</main>
reports/skill_atlas.json
+180 -38
View File
@@ -3,11 +3,41 @@
"workspace_root": ".",
"summary": {
"skill_count": 12,
"actionable_skill_count": 1,
"route_collision_count": 5,
"actionable_route_collision_count": 0,
"owner_gap_count": 9,
"actionable_owner_gap_count": 0,
"stale_count": 10,
"actionable_stale_count": 0,
"shared_resource_count": 0,
"no_route_opportunity_count": 0
"no_route_opportunity_count": 0,
"non_actionable_issue_count": 24
},
"scope_policy": {
"present": true,
"path": "skill_atlas/policy.json",
"schema_version": "1.0",
"rules": [
{
"path_prefix": "examples/",
"scope": "example",
"actionable": false,
"reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"path_prefix": "tests/fixtures/",
"scope": "test-fixture",
"actionable": false,
"reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers."
},
{
"path_prefix": "geo-ranking-article-generator",
"scope": "embedded-generated-skill",
"actionable": false,
"reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release."
}
]
},
"catalog": {
"workspace_root": ".",
@@ -30,11 +60,13 @@
"agent-skills-compatible"
],
"resources": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
"scripts/build_skill_atlas.py",
"scripts/check_update.py",
"scripts/ci_test.py",
"scripts/collect_feedback.py",
"scripts/compile_skill.py",
"scripts/context_sizer.py",
"scripts/create_iteration_snapshot.py",
"scripts/cross_packager.py",
@@ -45,8 +77,12 @@
"scripts/init_skill.py",
"scripts/judge_blind_eval.py",
"scripts/lint_skill.py",
"scripts/local_output_eval_runner.py",
"scripts/optimize_description.py",
"scripts/probe_runtime_permissions.py",
"scripts/promotion_checker.py",
"scripts/registry_audit.py",
"scripts/render_adoption_drift_report.py",
"scripts/render_artifact_design_profile.py",
"scripts/render_baseline_compare.py",
"scripts/render_context_reports.py",
@@ -62,8 +98,10 @@
"scripts/render_reference_scan.py",
"scripts/render_reference_synthesis.py",
"scripts/render_regression_history.py",
"scripts/render_review_annotations.py",
"scripts/render_review_studio.py",
"scripts/render_review_viewer.py",
"scripts/render_review_waivers.py",
"scripts/render_skill_overview.py",
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
@@ -72,16 +110,21 @@
"scripts/run_description_optimization_suite.py",
"scripts/run_eval_suite.py",
"scripts/run_output_eval.py",
"scripts/run_output_execution.py",
"scripts/simulate_install.py",
"scripts/skill_report_charts.py",
"scripts/skill_report_metrics.py",
"scripts/skill_report_model.py",
"scripts/sync_local_install.py",
"scripts/trigger_eval.py",
"scripts/trust_check.py",
"scripts/upgrade_check.py",
"scripts/validate_skill.py",
"scripts/verify_package.py",
"scripts/yao.py",
"references/artifact-design-doctrine.md",
"references/authoring-discipline.md",
"references/distribution-registry-method.md",
"references/eval-playbook.md",
"references/gate-selection.md",
"references/governance.md",
@@ -102,12 +145,14 @@
"references/regression-cause-taxonomy.md",
"references/resource-boundaries.md",
"references/review-studio-method.md",
"references/review-waiver-method.md",
"references/runtime-conformance-method.md",
"references/skill-archetypes.md",
"references/skill-atlas-method.md",
"references/skill-engineering-method.md",
"references/skill-ir-method.md",
"references/systems-thinking-doctrine.md",
"references/telemetry-drift-method.md",
"references/trust-security-method.md",
"templates/basic_skill.md.j2",
"templates/complex_skill.md.j2"
@@ -131,7 +176,10 @@
"team",
"transcripts",
"workflows"
]
],
"atlas_scope": "release",
"actionable": true,
"scope_reason": "default release-actionable skill"
},
{
"name": "release-orchestrator",
@@ -186,7 +234,10 @@
"stakeholder",
"theory",
"workflow"
]
],
"atlas_scope": "example",
"actionable": false,
"scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"name": "frontend-review",
@@ -222,7 +273,10 @@
"states",
"ui",
"ux"
]
],
"atlas_scope": "example",
"actionable": false,
"scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"name": "frontend-review",
@@ -242,7 +296,10 @@
"frontend",
"merge",
"review"
]
],
"atlas_scope": "example",
"actionable": false,
"scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"name": "frontend-review",
@@ -277,7 +334,10 @@
"security",
"states",
"ui"
]
],
"atlas_scope": "example",
"actionable": false,
"scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"name": "incident-command-governor",
@@ -315,7 +375,10 @@
"run",
"severity",
"standardize"
]
],
"atlas_scope": "example",
"actionable": false,
"scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"name": "note-cleanup",
@@ -349,7 +412,10 @@
"separate",
"structured",
"summaries"
]
],
"atlas_scope": "example",
"actionable": false,
"scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"name": "frontend-review",
@@ -386,7 +452,10 @@
"ui",
"unsafe",
"ux"
]
],
"atlas_scope": "example",
"actionable": false,
"scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package."
},
{
"name": "geo-ranking-article-generator",
@@ -443,7 +512,10 @@
"write",
"writing",
"生成式引擎优化"
]
],
"atlas_scope": "embedded-generated-skill",
"actionable": false,
"scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release."
},
{
"name": "invalid-governance-skill",
@@ -462,7 +534,10 @@
"governance",
"test",
"validation"
]
],
"atlas_scope": "test-fixture",
"actionable": false,
"scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers."
},
{
"name": "broken-yaml-skill",
@@ -483,7 +558,10 @@
"packager",
"testing",
"yaml"
]
],
"atlas_scope": "test-fixture",
"actionable": false,
"scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers."
},
{
"name": "broken-skill",
@@ -503,16 +581,24 @@
"fixture",
"packager",
"testing"
]
],
"atlas_scope": "test-fixture",
"actionable": false,
"scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers."
}
],
"summary": {
"skill_count": 12,
"actionable_skill_count": 1,
"route_collision_count": 5,
"actionable_route_collision_count": 0,
"owner_gap_count": 9,
"actionable_owner_gap_count": 0,
"stale_count": 10,
"actionable_stale_count": 0,
"shared_resource_count": 0,
"no_route_opportunity_count": 0
"no_route_opportunity_count": 0,
"non_actionable_issue_count": 24
}
},
"route_collisions": [
@@ -522,7 +608,10 @@
"path_a": "examples/evolution-frontend-review/final",
"path_b": "examples/evolution-frontend-review/v1",
"score": 0.708,
"status": "collision"
"status": "collision",
"actionable": false,
"scope_a": "example",
"scope_b": "example"
},
{
"skill_a": "frontend-review",
@@ -530,7 +619,10 @@
"path_a": "examples/evolution-frontend-review/final",
"path_b": "examples/team-frontend-review/generated-skill",
"score": 0.64,
"status": "collision"
"status": "collision",
"actionable": false,
"scope_a": "example",
"scope_b": "example"
},
{
"skill_a": "frontend-review",
@@ -538,7 +630,10 @@
"path_a": "examples/evolution-frontend-review/v1",
"path_b": "examples/team-frontend-review/generated-skill",
"score": 0.538,
"status": "collision"
"status": "collision",
"actionable": false,
"scope_a": "example",
"scope_b": "example"
},
{
"skill_a": "broken-yaml-skill",
@@ -546,7 +641,10 @@
"path_a": "tests/fixtures/package_invalid_yaml",
"path_b": "tests/fixtures/package_missing_interface_field",
"score": 0.833,
"status": "collision"
"status": "collision",
"actionable": false,
"scope_a": "test-fixture",
"scope_b": "test-fixture"
},
{
"skill_a": "frontend-review",
@@ -554,9 +652,13 @@
"path_a": "examples/evolution-frontend-review/final",
"path_b": "examples/evolution-frontend-review/v0",
"score": 1.0,
"status": "duplicate-name"
"status": "duplicate-name",
"actionable": false,
"scope_a": "example",
"scope_b": "example"
}
],
"actionable_route_collisions": [],
"dependency_graph": {
"nodes": [
{
@@ -614,56 +716,77 @@
{
"name": "frontend-review",
"path": "examples/evolution-frontend-review/final",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "example"
},
{
"name": "frontend-review",
"path": "examples/evolution-frontend-review/v0",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "example"
},
{
"name": "frontend-review",
"path": "examples/evolution-frontend-review/v1",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "example"
},
{
"name": "incident-command-governor",
"path": "examples/governed-incident-command/generated-skill",
"reason": "review overdue by cadence monthly",
"age_days": 74,
"allowed_days": 31
"allowed_days": 31,
"actionable": false,
"scope": "example"
},
{
"name": "note-cleanup",
"path": "examples/simple-note-cleanup/generated-skill",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "example"
},
{
"name": "frontend-review",
"path": "examples/team-frontend-review/generated-skill",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "example"
},
{
"name": "geo-ranking-article-generator",
"path": "geo-ranking-article-generator",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "embedded-generated-skill"
},
{
"name": "invalid-governance-skill",
"path": "tests/fixtures/governance_invalid_manifest",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "test-fixture"
},
{
"name": "broken-yaml-skill",
"path": "tests/fixtures/package_invalid_yaml",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "test-fixture"
},
{
"name": "broken-skill",
"path": "tests/fixtures/package_missing_interface_field",
"reason": "missing updated_at"
"reason": "missing updated_at",
"actionable": false,
"scope": "test-fixture"
}
],
"actionable_stale_skills": [],
"owner_review_gaps": [
{
"name": "frontend-review",
@@ -672,7 +795,9 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "example"
},
{
"name": "frontend-review",
@@ -681,7 +806,9 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "example"
},
{
"name": "frontend-review",
@@ -690,7 +817,9 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "example"
},
{
"name": "note-cleanup",
@@ -699,7 +828,9 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "example"
},
{
"name": "frontend-review",
@@ -708,7 +839,9 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "example"
},
{
"name": "geo-ranking-article-generator",
@@ -717,7 +850,9 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "embedded-generated-skill"
},
{
"name": "invalid-governance-skill",
@@ -725,7 +860,9 @@
"missing": [
"owner",
"review_cadence"
]
],
"actionable": false,
"scope": "test-fixture"
},
{
"name": "broken-yaml-skill",
@@ -734,7 +871,9 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "test-fixture"
},
{
"name": "broken-skill",
@@ -743,9 +882,12 @@
"owner",
"review_cadence",
"maturity"
]
],
"actionable": false,
"scope": "test-fixture"
}
],
"actionable_owner_review_gaps": [],
"no_route_opportunities": [],
"artifacts": {
"catalog": "skill_atlas/catalog.json",
reports/system-model.json
+31 -22
View File
@@ -3,21 +3,38 @@
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"systems_doctrine": "Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight.",
"stability": {
"score": 86,
"band": "stable-first-pass"
"score": 100,
"band": "system-ready"
},
"boundary_map": {
"owned_job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"owned_job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"input_boundary": [
"user-provided workflow notes, prompts, docs, or examples"
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"output_boundary": "a reusable skill output",
"output_boundary": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"non_goals": [
"one-off adjacent requests that do not match the recurring job",
"private local material that was not intentionally included"
"one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill",
"general code review or debugging unless the user is packaging that workflow as a skill",
"raw private material that was not intentionally supplied as skill evidence",
"platform-specific plugin creation when the request is not about an agent skill package"
],
"constraints": [
"keep SKILL.md lean and route primarily through frontmatter description",
"put durable guidance in references, executable logic in scripts, and evidence in reports",
"default to the lightest reliable mode before adding governance weight",
"preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets",
"avoid raw prompt, output, transcript, or private content in telemetry"
],
"standards": [
"trigger boundaries must be tested with should-trigger and should-not-trigger cases",
"production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence",
"governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation",
"generated reports should be bilingual or reviewer-friendly when they are user-facing",
"each new asset must earn its place by reducing ambiguity, risk, or repeated work"
],
"constraints": [],
"standards": [],
"human_judgment_boundary": [
"Ask one focused clarification when the real job, output, or exclusion boundary is unclear.",
"Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.",
@@ -28,7 +45,7 @@
"feedback_loops": [
{
"name": "Intent boundary loop",
"signal": "Intent confidence score is 30/100.",
"signal": "Intent confidence score is 100/100.",
"response": "Ask only the highest-leverage clarification before adding package weight.",
"evidence": "reports/intent-confidence.md and reports/intent-dialogue.md"
},
@@ -38,8 +55,8 @@
"response": "Borrow one pattern at a time and keep the rest as reviewer-visible evidence.",
"evidence": "reports/reference-synthesis.md",
"current_patterns": [
"Borrow a first-time operator flow that explains itself before it asks for more structure.",
"Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.",
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed.",
"Borrow the discipline of defining what the skill should not own before growing the package.",
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure."
@@ -131,18 +148,10 @@
"family": "Prompt-behavior mismatch",
"symptom": "The role, task, and format are copied from a prompt instead of becoming stable skill behavior.",
"repair": "Convert reusable role/task/format assumptions into workflow, reports, or references.",
"watch_axes": [
"Specificity"
]
"watch_axes": []
}
],
"leverage_points": [
{
"rank": 1,
"point": "Clarify the real job boundary",
"why": "Intent uncertainty creates downstream trigger, output, and governance errors.",
"move": "Ask one focused question and update intent context before adding assets."
},
{
"rank": 2,
"point": "Tune the frontmatter description",
@@ -159,7 +168,7 @@
"rank": 4,
"point": "Borrow one pattern, not a whole product",
"why": "External references improve quality when reduced to structure, not copied as surface style.",
"move": "Start from: Borrow a first-time operator flow that explains itself before it asks for more structure."
"move": "Start from: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts."
},
{
"rank": 5,
reports/system-model.md
+28 -18
View File
@@ -2,20 +2,37 @@
Skill: `yao-meta-skill`
- Stability score: `86/100`
- Stability band: `stable-first-pass`
- Stability score: `100/100`
- Stability band: `system-ready`
- Doctrine: Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight.
## System Boundary Map
- Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
- Output boundary: a reusable skill output
- Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.
- Output boundary: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.
- Maturity assumption: `governed`
- Input boundary:
- user-provided workflow notes, prompts, docs, or examples
- rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions
- an existing skill directory that needs refactor, evaluation, packaging, or governance hardening
- target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution
- benchmark references, local constraints, desired maturity tier, and review standards
- Non-goals:
- one-off adjacent requests that do not match the recurring job
- private local material that was not intentionally included
- one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill
- general code review or debugging unless the user is packaging that workflow as a skill
- raw private material that was not intentionally supplied as skill evidence
- platform-specific plugin creation when the request is not about an agent skill package
- Constraints:
- keep SKILL.md lean and route primarily through frontmatter description
- put durable guidance in references, executable logic in scripts, and evidence in reports
- default to the lightest reliable mode before adding governance weight
- preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets
- avoid raw prompt, output, transcript, or private content in telemetry
- Standards:
- trigger boundaries must be tested with should-trigger and should-not-trigger cases
- production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence
- governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation
- generated reports should be bilingual or reviewer-friendly when they are user-facing
- each new asset must earn its place by reducing ambiguity, risk, or repeated work
- Human judgment boundary:
- Ask one focused clarification when the real job, output, or exclusion boundary is unclear.
- Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.
@@ -25,7 +42,7 @@ Skill: `yao-meta-skill`
### Intent boundary loop
- Signal: Intent confidence score is 30/100.
- Signal: Intent confidence score is 100/100.
- Response: Ask only the highest-leverage clarification before adding package weight.
- Evidence: reports/intent-confidence.md and reports/intent-dialogue.md
@@ -35,8 +52,8 @@ Skill: `yao-meta-skill`
- Response: Borrow one pattern at a time and keep the rest as reviewer-visible evidence.
- Evidence: reports/reference-synthesis.md
- Current patterns:
- Borrow a first-time operator flow that explains itself before it asks for more structure.
- Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.
- Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.
- Borrow a review checkpoint wherever trust matters more than raw speed.
- Borrow the discipline of defining what the skill should not own before growing the package.
- Borrow the way it turns a messy workflow into a repeatable operating path.
- Borrow the clear execution entrypoints and command structure.
@@ -124,16 +141,9 @@ Skill: `yao-meta-skill`
- Symptom: The role, task, and format are copied from a prompt instead of becoming stable skill behavior.
- Repair: Convert reusable role/task/format assumptions into workflow, reports, or references.
- Watch Axes:
- Specificity
## Highest Leverage Moves
### 1. Clarify the real job boundary
- Why: Intent uncertainty creates downstream trigger, output, and governance errors.
- Move: Ask one focused question and update intent context before adding assets.
### 2. Tune the frontmatter description
- Why: The description is the highest-leverage routing surface.
@@ -147,7 +157,7 @@ Skill: `yao-meta-skill`
### 4. Borrow one pattern, not a whole product
- Why: External references improve quality when reduced to structure, not copied as surface style.
- Move: Start from: Borrow a first-time operator flow that explains itself before it asks for more structure.
- Move: Start from: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.
### 5. Close the lifecycle loop
reports/upgrade_check.json
+94
View File
@@ -0,0 +1,94 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13",
"previous": {
"name": "yao-meta-skill",
"version": "1.0.0",
"targets": [
"openai",
"claude",
"generic"
]
},
"current": {
"name": "yao-meta-skill",
"version": "1.1.0",
"targets": [
"openai",
"claude",
"generic",
"agent-skills-compatible"
]
},
"summary": {
"declared_bump": "minor",
"recommended_bump": "minor",
"breaking_change_count": 0,
"added_target_count": 1,
"metadata_change_count": 1,
"checksum_change_count": 2,
"failure_count": 0,
"warning_count": 0
},
"upgrade_diff": {
"name_changed": false,
"added_targets": [
"agent-skills-compatible"
],
"removed_targets": [],
"compatibility_changes": [
{
"target": "agent-skills",
"from": "missing",
"to": "pass",
"direction": "improved"
},
{
"target": "agent-skills-compatible",
"from": "missing",
"to": "pass",
"direction": "improved"
},
{
"target": "vscode",
"from": "missing",
"to": "pass",
"direction": "improved"
}
],
"metadata_changes": [
{
"field": "description",
"from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.",
"to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
}
],
"checksum_changes": [
{
"field": "archive_sha256",
"from": "",
"to": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe"
}
]
},
"release_notes": [
"Recommended version bump: minor.",
"Added targets: agent-skills-compatible.",
"Package or archive checksum changed; reviewers should verify package artifacts before release.",
"Package verification evidence: reports/package_verification.md."
],
"failures": [],
"warnings": [],
"artifacts": {
"previous_package": "registry/examples/yao-meta-skill-1.0.0.json",
"current_package": "reports/registry_audit.json",
"json": "reports/upgrade_check.json",
"markdown": "reports/upgrade_check.md"
}
}
reports/upgrade_check.md
+31
View File
@@ -0,0 +1,31 @@
# Upgrade Check
- OK: `True`
- Previous: `yao-meta-skill 1.0.0`
- Current: `yao-meta-skill 1.1.0`
- Declared bump: `minor`
- Recommended bump: `minor`
- Breaking changes: `0`
## Upgrade Diff
- Added targets: `agent-skills-compatible`
- Removed targets: `none`
- Compatibility changes: `3`
- Metadata changes: `1`
- Checksum changes: `2`
## Release Notes
- Recommended version bump: minor.
- Added targets: agent-skills-compatible.
- Package or archive checksum changed; reviewers should verify package artifacts before release.
- Package verification evidence: reports/package_verification.md.
## Failures
- None
## Warnings
- None
scripts/adjudicate_output_review.py
+351
View File
@@ -0,0 +1,351 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_BLIND_PACK = ROOT / "reports" / "output_blind_review_pack.json"
DEFAULT_ANSWER_KEY = ROOT / "reports" / "output_blind_answer_key.json"
DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json"
DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_adjudication.json"
DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_adjudication.md"
def display_path(path: Path) -> str:
try:
return str(path.resolve().relative_to(ROOT.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> tuple[dict[str, Any], list[str]]:
if not path.exists():
return {}, [f"Missing JSON file: {display_path(path)}"]
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return {}, [f"Invalid JSON file {display_path(path)}: {exc}"]
if not isinstance(payload, dict):
return {}, [f"JSON root must be an object: {display_path(path)}"]
return payload, []
def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]:
if not path.exists():
return {"schema_version": "1.0", "decisions": []}, []
return load_json(path)
def normalize_variant(value: Any) -> str:
normalized = str(value or "").strip().upper()
if normalized in {"VARIANT A", "A"}:
return "A"
if normalized in {"VARIANT B", "B"}:
return "B"
return normalized
def confidence_value(value: Any) -> tuple[float | None, str | None]:
if value is None or value == "":
return None, None
try:
parsed = float(value)
except (TypeError, ValueError):
return None, f"confidence must be numeric, got {value!r}"
if parsed < 0 or parsed > 1:
return None, f"confidence must be between 0 and 1, got {parsed}"
return round(parsed, 3), None
def answer_index(answer_key: dict[str, Any]) -> dict[str, dict[str, Any]]:
answers = answer_key.get("answers", [])
if not isinstance(answers, list):
return {}
return {str(item.get("case_id", "")): item for item in answers if isinstance(item, dict) and item.get("case_id")}
def pair_index(blind_pack: dict[str, Any]) -> dict[str, dict[str, Any]]:
pairs = blind_pack.get("pairs", [])
if not isinstance(pairs, list):
return {}
return {str(item.get("case_id", "")): item for item in pairs if isinstance(item, dict) and item.get("case_id")}
def decision_index(decisions_payload: dict[str, Any]) -> tuple[dict[str, dict[str, Any]], list[str]]:
decisions = decisions_payload.get("decisions", [])
failures: list[str] = []
indexed: dict[str, dict[str, Any]] = {}
if not isinstance(decisions, list):
return {}, ["decisions must be a list"]
for index, item in enumerate(decisions, start=1):
if not isinstance(item, dict):
failures.append(f"decision #{index} must be an object")
continue
case_id = str(item.get("case_id", "")).strip()
if not case_id:
failures.append(f"decision #{index} is missing case_id")
continue
if case_id in indexed:
failures.append(f"duplicate decision for case_id: {case_id}")
continue
indexed[case_id] = item
return indexed, failures
def build_decision_template(blind_pack: dict[str, Any]) -> dict[str, Any]:
pairs = blind_pack.get("pairs", [])
template_decisions = []
if isinstance(pairs, list):
for pair in pairs:
if not isinstance(pair, dict):
continue
template_decisions.append(
{
"case_id": str(pair.get("case_id", "")),
"winner_variant": "",
"confidence": None,
"reason": "",
}
)
return {
"schema_version": "1.0",
"reviewer": "",
"reviewed_at": "",
"decision_contract": {
"winner_variant": "Use A or B after reading the blind review pack. Leave blank when pending.",
"confidence": "Optional number from 0 to 1.",
"reason": "Short reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.",
},
"decisions": template_decisions,
}
def adjudicate_pair(
case_id: str,
pair: dict[str, Any],
answer: dict[str, Any] | None,
decision: dict[str, Any] | None,
) -> tuple[dict[str, Any], list[str]]:
failures: list[str] = []
expected = normalize_variant(answer.get("expected_winner_variant", "") if answer else "")
if expected not in {"A", "B"}:
failures.append(f"{case_id}: answer key is missing expected_winner_variant")
if decision is None:
return (
{
"case_id": case_id,
"status": "pending",
"expected_winner_variant": expected,
"reviewer_winner_variant": "",
"confidence": None,
"reason": "",
"prompt": str(pair.get("prompt", "")),
},
failures,
)
reviewer = normalize_variant(
decision.get("winner_variant", decision.get("reviewer_winner_variant", decision.get("winner", "")))
)
confidence, confidence_failure = confidence_value(decision.get("confidence"))
reason = str(decision.get("reason", "")).strip()
if not reviewer:
return (
{
"case_id": case_id,
"status": "pending",
"expected_winner_variant": expected,
"reviewer_winner_variant": "",
"confidence": confidence,
"reason": reason,
"prompt": str(pair.get("prompt", "")),
},
failures,
)
if reviewer not in {"A", "B"}:
failures.append(f"{case_id}: winner_variant must be A or B")
status = "invalid"
elif confidence_failure:
failures.append(f"{case_id}: {confidence_failure}")
status = "invalid"
else:
status = "match" if reviewer == expected else "disagree"
return (
{
"case_id": case_id,
"status": status,
"expected_winner_variant": expected,
"reviewer_winner_variant": reviewer,
"confidence": confidence,
"reason": reason,
"prompt": str(pair.get("prompt", "")),
},
failures,
)
def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, Any]:
pair_count = len(pairs)
judgment_count = sum(1 for item in pairs if item["status"] in {"match", "disagree"})
pending_count = sum(1 for item in pairs if item["status"] == "pending")
agreement_count = sum(1 for item in pairs if item["status"] == "match")
disagreement_count = sum(1 for item in pairs if item["status"] == "disagree")
invalid_decision_count = sum(1 for item in pairs if item["status"] == "invalid")
agreement_rate = round(agreement_count / judgment_count * 100, 2) if judgment_count else None
return {
"pair_count": pair_count,
"judgment_count": judgment_count,
"pending_count": pending_count,
"agreement_count": agreement_count,
"disagreement_count": disagreement_count,
"invalid_decision_count": invalid_decision_count,
"agreement_rate": agreement_rate,
"needs_review": pending_count > 0,
"failure_count": len(failures),
}
def render_markdown(payload: dict[str, Any]) -> str:
summary = payload["summary"]
lines = [
"# Output Review Adjudication",
"",
"This report adjudicates reviewer choices from the blind A/B output review pack against the separate answer key.",
"",
f"- Pairs: `{summary['pair_count']}`",
f"- Judgments: `{summary['judgment_count']}`",
f"- Pending: `{summary['pending_count']}`",
f"- Agreement rate: `{summary['agreement_rate'] if summary['agreement_rate'] is not None else 'n/a'}`",
f"- Invalid decisions: `{summary['invalid_decision_count']}`",
"",
]
if summary["judgment_count"] == 0:
lines.extend(
[
"No reviewer decisions recorded yet.",
"",
"Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.",
"",
]
)
lines.extend(
[
"## Case Adjudication",
"",
"| Case | Reviewer | Expected | Status | Confidence | Reason |",
"| --- | --- | --- | --- | ---: | --- |",
]
)
for item in payload["pairs"]:
confidence = "" if item.get("confidence") is None else str(item["confidence"])
reason = str(item.get("reason", "")).replace("|", "\\|") or ""
lines.append(
f"| {item['case_id']} | {item.get('reviewer_winner_variant', '') or 'pending'} | "
f"{item.get('expected_winner_variant', '') or 'missing'} | {item['status']} | {confidence} | {reason} |"
)
if payload.get("failures"):
lines.extend(["", "## Failures", ""])
for failure in payload["failures"]:
lines.append(f"- {failure}")
lines.extend(
[
"",
"## Next Fixes",
"",
"- Keep the blind review pack separate from the answer key until decisions are recorded.",
"- Treat disagreement cases as prompts for rubric tuning or output improvement.",
"- Add model-executed holdout runs after this human adjudication harness is stable.",
]
)
return "\n".join(lines).strip() + "\n"
def adjudicate_output_review(
blind_pack_path: Path,
answer_key_path: Path,
decisions_path: Path,
output_json: Path,
output_md: Path,
write_template: bool = False,
) -> dict[str, Any]:
blind_pack, failures = load_json(blind_pack_path)
answer_key, answer_failures = load_json(answer_key_path)
failures.extend(answer_failures)
decisions_payload, decision_failures = load_optional_decisions(decisions_path)
failures.extend(decision_failures)
template_written = False
if write_template and blind_pack and not decisions_path.exists():
decisions_path.parent.mkdir(parents=True, exist_ok=True)
decisions_path.write_text(
json.dumps(build_decision_template(blind_pack), ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
template_written = True
pairs_by_id = pair_index(blind_pack)
answers_by_id = answer_index(answer_key)
decisions_by_id, index_failures = decision_index(decisions_payload)
failures.extend(index_failures)
for case_id in decisions_by_id:
if case_id not in pairs_by_id:
failures.append(f"decision references unknown case_id: {case_id}")
adjudicated_pairs: list[dict[str, Any]] = []
for case_id, pair in pairs_by_id.items():
adjudicated, pair_failures = adjudicate_pair(case_id, pair, answers_by_id.get(case_id), decisions_by_id.get(case_id))
adjudicated_pairs.append(adjudicated)
failures.extend(pair_failures)
summary = build_summary(adjudicated_pairs, failures)
payload = {
"schema_version": "1.0",
"ok": not failures,
"summary": summary,
"reviewer": decisions_payload.get("reviewer", ""),
"reviewed_at": decisions_payload.get("reviewed_at", ""),
"artifacts": {
"blind_pack": display_path(blind_pack_path),
"answer_key": display_path(answer_key_path),
"decisions": display_path(decisions_path),
"json": display_path(output_json),
"markdown": display_path(output_md),
},
"template_written": template_written,
"pairs": adjudicated_pairs,
"failures": failures,
}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(payload), encoding="utf-8")
return payload
def main() -> None:
parser = argparse.ArgumentParser(description="Adjudicate blind A/B output review decisions against the answer key.")
parser.add_argument("--blind-pack", default=str(DEFAULT_BLIND_PACK))
parser.add_argument("--answer-key", default=str(DEFAULT_ANSWER_KEY))
parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS))
parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON))
parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD))
parser.add_argument("--write-template", action="store_true")
args = parser.parse_args()
payload = adjudicate_output_review(
Path(args.blind_pack).resolve(),
Path(args.answer_key).resolve(),
Path(args.decisions).resolve(),
Path(args.output_json).resolve(),
Path(args.output_md).resolve(),
write_template=args.write_template,
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 2)
if __name__ == "__main__":
main()
scripts/build_skill_atlas.py
+115 -12
View File
@@ -52,6 +52,11 @@ CADENCE_DAYS = {
"annual": 370,
"per-release": 120,
}
DEFAULT_SCOPE = {
"scope": "release",
"actionable": True,
"scope_reason": "default release-actionable skill",
}
def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]:
@@ -86,6 +91,20 @@ def load_json(path: Path) -> dict[str, Any]:
return payload if isinstance(payload, dict) else {}
def load_scope_policy(workspace_root: Path) -> dict[str, Any]:
path = workspace_root / "skill_atlas" / "policy.json"
if not path.exists():
return {"present": False, "path": safe_rel(workspace_root, path), "rules": []}
payload = load_json(path)
rules = payload.get("scope_rules", [])
return {
"present": True,
"path": safe_rel(workspace_root, path),
"schema_version": str(payload.get("schema_version", "")),
"rules": rules if isinstance(rules, list) else [],
}
def should_skip(path: Path, root: Path) -> bool:
try:
rel = path.relative_to(root)
@@ -124,6 +143,29 @@ def safe_rel(root: Path, path: Path) -> str:
return str(path.resolve())
def path_matches_prefix(rel_path: str, prefix: str) -> bool:
normalized_path = rel_path.strip("/")
normalized_prefix = prefix.strip("/")
if not normalized_prefix:
return False
return normalized_path == normalized_prefix or normalized_path.startswith(normalized_prefix + "/")
def scope_for_path(rel_path: str, policy: dict[str, Any]) -> dict[str, Any]:
for rule in policy.get("rules", []):
if not isinstance(rule, dict):
continue
prefix = str(rule.get("path_prefix", "")).strip()
if not prefix or not path_matches_prefix(rel_path, prefix):
continue
return {
"scope": str(rule.get("scope") or "supporting"),
"actionable": bool(rule.get("actionable", False)),
"scope_reason": str(rule.get("reason") or f"matched policy prefix {prefix}"),
}
return dict(DEFAULT_SCOPE)
def display_path(path: Path) -> str:
try:
return str(path.resolve().relative_to(ROOT.resolve()))
@@ -148,15 +190,17 @@ def resource_names(skill_dir: Path) -> list[str]:
return names
def collect_skill(workspace_root: Path, skill_dir: Path) -> dict[str, Any]:
def collect_skill(workspace_root: Path, skill_dir: Path, policy: dict[str, Any]) -> dict[str, Any]:
frontmatter, _ = parse_frontmatter(skill_dir / "SKILL.md")
manifest = load_json(skill_dir / "manifest.json")
name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name)
description = str(frontmatter.get("description") or "")
targets = manifest.get("target_platforms", [])
rel_path = safe_rel(workspace_root, skill_dir)
scope = scope_for_path(rel_path, policy)
return {
"name": name,
"path": safe_rel(workspace_root, skill_dir),
"path": rel_path,
"description": description,
"owner": str(manifest.get("owner", "")),
"version": str(manifest.get("version", "")),
@@ -167,6 +211,9 @@ def collect_skill(workspace_root: Path, skill_dir: Path) -> dict[str, Any]:
"targets": [str(item) for item in targets] if isinstance(targets, list) else [],
"resources": resource_names(skill_dir),
"token_set": sorted(tokens(description)),
"atlas_scope": scope["scope"],
"actionable": scope["actionable"],
"scope_reason": scope["scope_reason"],
}
@@ -184,6 +231,9 @@ def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[
"path_b": right["path"],
"score": score,
"status": status,
"actionable": bool(left.get("actionable") and right.get("actionable")),
"scope_a": str(left.get("atlas_scope", "")),
"scope_b": str(right.get("atlas_scope", "")),
}
rows.append(row)
if status == "collision":
@@ -202,6 +252,19 @@ def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[
"path_b": item["paths"][1],
"score": 1.0,
"status": "duplicate-name",
"actionable": all(
skill.get("actionable")
for skill in skills
if skill["name"] == item["name"] and skill["path"] in set(item["paths"][:2])
),
"scope_a": next(
(str(skill.get("atlas_scope", "")) for skill in skills if skill["path"] == item["paths"][0]),
"",
),
"scope_b": next(
(str(skill.get("atlas_scope", "")) for skill in skills if skill["path"] == item["paths"][1]),
"",
),
}
)
return rows, collisions
@@ -239,7 +302,15 @@ def stale_skills(skills: list[dict[str, Any]], today: date) -> list[dict[str, An
cadence = skill.get("review_cadence") or ""
allowed_days = CADENCE_DAYS.get(cadence, 120)
if not updated:
stale.append({"name": skill["name"], "path": skill["path"], "reason": "missing updated_at"})
stale.append(
{
"name": skill["name"],
"path": skill["path"],
"reason": "missing updated_at",
"actionable": bool(skill.get("actionable")),
"scope": str(skill.get("atlas_scope", "")),
}
)
continue
age = (today - updated).days
if age > allowed_days:
@@ -250,6 +321,8 @@ def stale_skills(skills: list[dict[str, Any]], today: date) -> list[dict[str, An
"reason": f"review overdue by cadence {cadence or 'unspecified'}",
"age_days": age,
"allowed_days": allowed_days,
"actionable": bool(skill.get("actionable")),
"scope": str(skill.get("atlas_scope", "")),
}
)
return stale
@@ -266,7 +339,15 @@ def owner_review_gaps(skills: list[dict[str, Any]]) -> list[dict[str, Any]]:
if not skill.get("maturity"):
missing.append("maturity")
if missing:
gaps.append({"name": skill["name"], "path": skill["path"], "missing": missing})
gaps.append(
{
"name": skill["name"],
"path": skill["path"],
"missing": missing,
"actionable": bool(skill.get("actionable")),
"scope": str(skill.get("atlas_scope", "")),
}
)
return gaps
@@ -288,7 +369,7 @@ def no_route_opportunities(workspace_root: Path) -> list[dict[str, Any]]:
def write_csv(path: Path, rows: list[dict[str, Any]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
fields = ["skill_a", "skill_b", "path_a", "path_b", "score", "status"]
fields = ["skill_a", "skill_b", "path_a", "path_b", "score", "status", "actionable", "scope_a", "scope_b"]
buffer = StringIO()
writer = csv.DictWriter(buffer, fieldnames=fields, lineterminator="\n")
writer.writeheader()
@@ -308,9 +389,10 @@ def render_html(payload: dict[str, Any]) -> str:
f"<td>{html.escape(skill.get('owner') or 'missing')}</td>"
f"<td>{html.escape(skill.get('maturity') or 'unknown')}</td>"
f"<td>{html.escape(skill.get('review_cadence') or 'missing')}</td>"
f"<td>{html.escape(skill.get('atlas_scope') or 'release')}</td>"
"</tr>"
)
blockers = payload["route_collisions"][:20] + payload["owner_review_gaps"][:20] + payload["stale_skills"][:20]
blockers = payload["actionable_route_collisions"][:20] + payload["actionable_owner_review_gaps"][:20] + payload["actionable_stale_skills"][:20]
blocker_items = "".join(
f"<li><strong>{html.escape(item.get('name', item.get('skill_a', 'issue')))}</strong> {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', [])))))}</li>"
for item in blockers
@@ -342,19 +424,24 @@ def render_html(payload: dict[str, Any]) -> str:
<p>Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.</p>
<section class="grid">
<div class="card"><span>Skills</span><strong>{summary['skill_count']}</strong></div>
<div class="card"><span>Route Collisions</span><strong>{summary['route_collision_count']}</strong></div>
<div class="card"><span>Owner Gaps</span><strong>{summary['owner_gap_count']}</strong></div>
<div class="card"><span>Stale Skills</span><strong>{summary['stale_count']}</strong></div>
<div class="card"><span>Actionable</span><strong>{summary['actionable_skill_count']}</strong></div>
<div class="card"><span>Route Collisions</span><strong>{summary['actionable_route_collision_count']}</strong></div>
<div class="card"><span>Owner Gaps</span><strong>{summary['actionable_owner_gap_count']}</strong></div>
<div class="card"><span>Stale Skills</span><strong>{summary['actionable_stale_count']}</strong></div>
<div class="card"><span>No-Route Opportunities</span><strong>{summary['no_route_opportunity_count']}</strong></div>
</section>
<section>
<h2>Top Issues</h2>
<h2>Actionable Issues</h2>
<ul>{blocker_items or '<li>No blocking portfolio issues detected.</li>'}</ul>
</section>
<section>
<h2>Full Portfolio Counts</h2>
<p>All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, and {summary['stale_count']} total stale signals.</p>
</section>
<section>
<h2>Catalog</h2>
<table>
<thead><tr><th>Name</th><th>Path</th><th>Owner</th><th>Maturity</th><th>Review</th></tr></thead>
<thead><tr><th>Name</th><th>Path</th><th>Owner</th><th>Maturity</th><th>Review</th><th>Scope</th></tr></thead>
<tbody>{''.join(rows)}</tbody>
</table>
</section>
@@ -366,20 +453,32 @@ def render_html(payload: dict[str, Any]) -> str:
def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, report_json: Path, threshold: float, today: date) -> dict[str, Any]:
workspace_root = workspace_root.resolve()
scope_policy = load_scope_policy(workspace_root)
skill_dirs = find_skill_dirs(workspace_root)
skills = [collect_skill(workspace_root, skill_dir) for skill_dir in skill_dirs]
skills = [collect_skill(workspace_root, skill_dir, scope_policy) for skill_dir in skill_dirs]
overlap_rows, collisions = route_overlap(skills, threshold)
graph = dependency_graph(skills)
stale = stale_skills(skills, today)
owner_gaps = owner_review_gaps(skills)
opportunities = no_route_opportunities(workspace_root)
actionable_skills = [skill for skill in skills if skill.get("actionable")]
actionable_collisions = [item for item in collisions if item.get("actionable")]
actionable_stale = [item for item in stale if item.get("actionable")]
actionable_owner_gaps = [item for item in owner_gaps if item.get("actionable")]
summary = {
"skill_count": len(skills),
"actionable_skill_count": len(actionable_skills),
"route_collision_count": len(collisions),
"actionable_route_collision_count": len(actionable_collisions),
"owner_gap_count": len(owner_gaps),
"actionable_owner_gap_count": len(actionable_owner_gaps),
"stale_count": len(stale),
"actionable_stale_count": len(actionable_stale),
"shared_resource_count": len(graph["shared_resources"]),
"no_route_opportunity_count": len(opportunities),
"non_actionable_issue_count": (len(collisions) - len(actionable_collisions))
+ (len(owner_gaps) - len(actionable_owner_gaps))
+ (len(stale) - len(actionable_stale)),
}
catalog = {
"workspace_root": display_path(workspace_root),
@@ -391,11 +490,15 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor
"ok": True,
"workspace_root": display_path(workspace_root),
"summary": summary,
"scope_policy": scope_policy,
"catalog": catalog,
"route_collisions": collisions,
"actionable_route_collisions": actionable_collisions,
"dependency_graph": graph,
"stale_skills": stale,
"actionable_stale_skills": actionable_stale,
"owner_review_gaps": owner_gaps,
"actionable_owner_review_gaps": actionable_owner_gaps,
"no_route_opportunities": opportunities,
"artifacts": {
"catalog": display_path(output_dir / "catalog.json"),
scripts/ci_test.py
+11
View File
@@ -34,13 +34,24 @@ DEFAULT_TARGETS = [
"skill-report-metrics-check",
"skill-report-charts-check",
"skill-ir-check",
"compiler-check",
"output-eval-check",
"output-execution-check",
"output-review-adjudication-check",
"runtime-conformance-check",
"runtime-permission-check",
"trust-check",
"skill-atlas-check",
"registry-audit-check",
"package-verify-check",
"install-simulation-check",
"upgrade-check",
"review-viewer-check",
"review-studio-check",
"feedback-check",
"adoption-drift-check",
"review-waivers-check",
"review-annotations-check",
"baseline-compare-check",
"reference-scan-check",
"github-benchmark-scan-check",
scripts/compile_skill.py
+726
View File
@@ -0,0 +1,726 @@
#!/usr/bin/env python3
import argparse
import json
from datetime import date
from pathlib import Path
from typing import Any
try:
import yaml
except ImportError: # pragma: no cover
yaml = None
ROOT = Path(__file__).resolve().parent.parent
COMPILER_SCHEMA_VERSION = "1.0"
COMPILER_NAME = "yao-skill-ir-compiler"
TARGET_TRANSFORMS: dict[str, dict[str, Any]] = {
"openai": {
"adapter_mode": "metadata-adapter",
"generated_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
"metadata_mapping": {
"display_name": "targets/openai/agents/openai.yaml::interface.display_name",
"default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt",
"activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode",
"execution": "targets/openai/agents/openai.yaml::compatibility.execution_context",
"trust": "targets/openai/agents/openai.yaml::compatibility.trust_level",
"permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract",
"degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy",
},
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
"unsupported_features": ["client-native script permission prompts are represented as permission contract metadata"],
},
"claude": {
"adapter_mode": "neutral-source-plus-adapter",
"generated_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
"metadata_mapping": {
"display_name": "targets/claude/adapter.json::display_name",
"default_prompt": "targets/claude/adapter.json::default_prompt",
"activation": "targets/claude/adapter.json::activation_mode",
"execution": "targets/claude/adapter.json::execution_context",
"trust": "targets/claude/adapter.json::trust_level",
"permissions": "targets/claude/adapter.json::target_permission_contract",
"degradation": "targets/claude/adapter.json::degradation_strategy",
},
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
"unsupported_features": ["vendor-native metadata fields are carried as adapter JSON and README notes"],
},
"generic": {
"adapter_mode": "agent-skills-compatible",
"generated_files": ["targets/generic/adapter.json"],
"metadata_mapping": {
"display_name": "targets/generic/adapter.json::display_name",
"default_prompt": "targets/generic/adapter.json::default_prompt",
"activation": "targets/generic/adapter.json::activation_mode",
"execution": "targets/generic/adapter.json::execution_context",
"trust": "targets/generic/adapter.json::trust_level",
"permissions": "targets/generic/adapter.json::target_permission_contract",
"degradation": "targets/generic/adapter.json::degradation_strategy",
},
"preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
"unsupported_features": [],
},
"agent-skills-compatible": {
"adapter_mode": "neutral-agent-skills-source",
"generated_files": ["SKILL.md", "agents/interface.yaml"],
"metadata_mapping": {
"name": "SKILL.md::frontmatter.name",
"description": "SKILL.md::frontmatter.description",
"interface": "agents/interface.yaml",
"manifest": "manifest.json",
},
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
"unsupported_features": [],
},
"agent-skills": {
"adapter_mode": "neutral-agent-skills-source",
"generated_files": ["SKILL.md", "agents/interface.yaml"],
"metadata_mapping": {
"name": "SKILL.md::frontmatter.name",
"description": "SKILL.md::frontmatter.description",
"interface": "agents/interface.yaml",
"manifest": "manifest.json",
},
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
"unsupported_features": [],
},
"vscode": {
"adapter_mode": "agent-skills-compatible-project-or-user-scope",
"generated_files": ["SKILL.md", "agents/interface.yaml"],
"metadata_mapping": {
"name": "folder-name-and-SKILL.md::frontmatter.name",
"description": "SKILL.md::frontmatter.description",
"interface": "agents/interface.yaml",
},
"preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"],
"unsupported_features": ["VS Code installation scope is documented but not installed by this compiler"],
},
}
def display_path(path: Path, root: Path = ROOT) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def load_yaml(path: Path) -> dict[str, Any]:
if not path.exists() or yaml is None:
return {}
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
return payload if isinstance(payload, dict) else {}
def read_frontmatter(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
text = path.read_text(encoding="utf-8", errors="replace")
if not text.startswith("---"):
return {}
parts = text.split("---", 2)
if len(parts) < 3:
return {}
if yaml is not None:
payload = yaml.safe_load(parts[1]) or {}
return payload if isinstance(payload, dict) else {}
data: dict[str, Any] = {}
for line in parts[1].splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
data[key.strip()] = value.strip().strip('"')
return data
def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]:
candidates = [
skill_dir / "reports" / "skill-ir.json",
skill_dir / "skill-ir" / "examples" / f"{name}.json",
skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json",
]
seen: set[Path] = set()
for path in candidates:
if path in seen:
continue
seen.add(path)
payload = load_json(path)
if payload:
return payload, display_path(path, skill_dir)
return {}, "frontmatter-fallback"
def count_list(payload: dict[str, Any], key: str) -> int:
value = payload.get(key, [])
return len(value) if isinstance(value, list) else 0
def list_or_empty(payload: dict[str, Any], key: str) -> list[str]:
value = payload.get(key, [])
return [str(item) for item in value] if isinstance(value, list) else []
def resource_counts(resources: dict[str, Any]) -> dict[str, int]:
return {key: count_list(resources, key) for key in ("references", "scripts", "assets", "reports")}
def eval_counts(eval_plan: dict[str, Any]) -> dict[str, int]:
return {
"trigger": count_list(eval_plan, "trigger"),
"output": count_list(eval_plan, "output"),
"adversarial": count_list(eval_plan, "adversarial"),
"baseline": 1 if eval_plan.get("baseline") else 0,
}
def scripts_with_flag(scripts: list[dict[str, Any]], flag: str) -> list[str]:
return [str(item.get("path")) for item in scripts if item.get(flag)]
def capability(name: str, scripts: list[str], review_reason: str) -> dict[str, Any]:
return {
"required": bool(scripts),
"script_count": len(scripts),
"scripts": scripts,
"review_reason": review_reason if scripts else "",
}
def permission_contract(skill_dir: Path) -> dict[str, Any]:
trust_path = skill_dir / "reports" / "security_trust_report.json"
trust = load_json(trust_path)
scripts = trust.get("scripts", []) if isinstance(trust.get("scripts"), list) else []
network_scripts = scripts_with_flag(scripts, "uses_network")
file_write_scripts = scripts_with_flag(scripts, "uses_file_write")
subprocess_scripts = scripts_with_flag(scripts, "uses_subprocess")
interactive_scripts = scripts_with_flag(scripts, "uses_input")
capabilities = {
"network": capability("network", network_scripts, "Outbound hosts must match security/network_policy.json."),
"file_write": capability("file_write", file_write_scripts, "Scripts write local files or generated artifacts."),
"subprocess": capability("subprocess", subprocess_scripts, "Scripts spawn local commands and need operator review."),
"interactive": capability("interactive", interactive_scripts, "Scripts prompt for user input or credentials."),
}
declared = [name for name, item in capabilities.items() if item["required"]]
help_smoke = trust.get("help_smoke", {}) if isinstance(trust.get("help_smoke"), dict) else {}
network_policy = trust.get("network_policy", {}) if isinstance(trust.get("network_policy"), dict) else {}
summary = trust.get("summary", {}) if isinstance(trust.get("summary"), dict) else {}
return {
"schema_version": "1.0",
"source": display_path(trust_path, skill_dir) if trust else "missing-security-trust-report",
"source_available": bool(trust),
"declared_capabilities": declared,
"review_required": bool(declared),
"capabilities": capabilities,
"network_policy": {
"source": network_policy.get("path", "security/network_policy.json"),
"covered_scripts": network_policy.get("covered_scripts", []),
"missing_scripts": network_policy.get("missing_scripts", []),
"mismatches": network_policy.get("mismatches", []),
},
"help_smoke": {
"enabled": bool(help_smoke.get("enabled")),
"checked_count": int(help_smoke.get("checked_count", 0) or 0),
"failed_count": int(help_smoke.get("failed_count", 0) or 0),
"failed_scripts": help_smoke.get("failed_scripts", []),
},
"trust_summary": {
"secret_findings": int(summary.get("secret_findings", 0) or 0),
"network_script_count": int(summary.get("network_script_count", len(network_scripts)) or 0),
"file_write_script_count": int(summary.get("file_write_script_count", len(file_write_scripts)) or 0),
"subprocess_script_count": len(subprocess_scripts),
"interactive_script_count": int(summary.get("interactive_script_count", len(interactive_scripts)) or 0),
"help_smoke_failed_count": int(summary.get("help_smoke_failed_count", 0) or 0),
},
}
TARGET_PERMISSION_MODELS = {
"openai": {
"model": "metadata-only",
"native_enforcement": False,
"representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json",
"operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package.",
},
"claude": {
"model": "neutral-source-plus-adapter",
"native_enforcement": False,
"representation": "targets/claude/adapter.json::target_permission_contract and README notes",
"operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review.",
},
"generic": {
"model": "agent-skills-compatible-metadata",
"native_enforcement": False,
"representation": "targets/generic/adapter.json::target_permission_contract",
"operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review.",
},
}
TARGET_NATIVE_MODELS = {
"openai": {
"native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source",
"activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.",
"resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.",
"script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.",
"permission_enforcement": "metadata-only",
"install_scope": "plugin or skill package consumer",
"review_artifacts": ["targets/openai/agents/openai.yaml", "targets/openai/adapter.json", "reports/review-studio.html"],
"fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.",
"unsupported_native_features": [
"client-native permission prompts",
"provider-executed scripts",
],
},
"claude": {
"native_surface": "Claude-compatible neutral source folder with adapter notes",
"activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.",
"resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.",
"script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.",
"permission_enforcement": "metadata-fallback",
"install_scope": "user or project skill directory",
"review_artifacts": ["targets/claude/README.md", "targets/claude/adapter.json", "reports/review-studio.html"],
"fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.",
"unsupported_native_features": [
"vendor-native permission enforcement",
"provider-specific execution transforms",
],
},
"generic": {
"native_surface": "Agent Skills compatible neutral package",
"activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.",
"resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.",
"script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.",
"permission_enforcement": "consumer-enforced-or-metadata-only",
"install_scope": "generic Agent Skills compatible root",
"review_artifacts": ["targets/generic/adapter.json", "reports/review-studio.html"],
"fallback_behavior": "Neutral source is the runtime fallback.",
"unsupported_native_features": [],
},
"agent-skills-compatible": {
"native_surface": "Agent Skills standard source tree",
"activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.",
"resource_strategy": "Keep optional directories as relative resources next to SKILL.md.",
"script_strategy": "Scripts remain local optional resources and should advertise --help when executable.",
"permission_enforcement": "consumer-enforced-or-metadata-only",
"install_scope": "Agent Skills source root",
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
"fallback_behavior": "The source tree itself is the target artifact.",
"unsupported_native_features": [],
},
"agent-skills": {
"native_surface": "Agent Skills standard source tree",
"activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.",
"resource_strategy": "Keep optional directories as relative resources next to SKILL.md.",
"script_strategy": "Scripts remain local optional resources and should advertise --help when executable.",
"permission_enforcement": "consumer-enforced-or-metadata-only",
"install_scope": "Agent Skills source root",
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
"fallback_behavior": "The source tree itself is the target artifact.",
"unsupported_native_features": [],
},
"vscode": {
"native_surface": "VS Code/Copilot Agent Skills project or user scope",
"activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.",
"resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.",
"script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.",
"permission_enforcement": "client-or-workspace-trust",
"install_scope": "VS Code user or project skills directory",
"review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"],
"fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.",
"unsupported_native_features": [
"automatic VS Code installation",
],
},
}
def target_permission_contract(target: str, permissions: dict[str, Any]) -> dict[str, Any]:
model = TARGET_PERMISSION_MODELS.get(
target,
{
"model": "metadata-only",
"native_enforcement": False,
"representation": "adapter metadata",
"operator_note": "Permission semantics are preserved as metadata for reviewer visibility.",
},
)
return {
"schema_version": "1.0",
"target": target,
"permission_model": model["model"],
"native_enforcement": model["native_enforcement"],
"representation": model["representation"],
"review_required": bool(permissions.get("review_required")),
"declared_capabilities": permissions.get("declared_capabilities", []),
"capability_counts": {
name: item.get("script_count", 0)
for name, item in permissions.get("capabilities", {}).items()
},
"evidence": permissions.get("source", ""),
"operator_note": model["operator_note"],
}
def target_native_contract(
target: str,
profile: dict[str, Any],
contract: dict[str, Any],
target_permissions: dict[str, Any],
) -> dict[str, Any]:
model = TARGET_NATIVE_MODELS.get(
target,
{
"native_surface": "adapter metadata",
"activation_policy": "Carry activation semantics as metadata for the target consumer.",
"resource_strategy": "Preserve neutral package resources.",
"script_strategy": "Expose script metadata for reviewer visibility.",
"permission_enforcement": "metadata-only",
"install_scope": "target consumer",
"review_artifacts": ["adapter.json", "reports/review-studio.html"],
"fallback_behavior": "Use neutral source package.",
"unsupported_native_features": [],
},
)
return {
"schema_version": "1.0",
"target": target,
"native_surface": model["native_surface"],
"activation": {
"policy": model["activation_policy"],
"trigger_description": contract.get("trigger", {}).get("description", ""),
"manual_activation_supported": True,
"automatic_activation_note": "Depends on the target client route/catalog implementation.",
},
"resources": {
"strategy": model["resource_strategy"],
"counts": contract.get("resources", {}).get("counts", {}),
"generated_files": profile.get("generated_files", []),
},
"scripts": {
"strategy": model["script_strategy"],
"script_count": contract.get("resources", {}).get("counts", {}).get("scripts", 0),
"help_smoke_failed_count": contract.get("permissions", {}).get("help_smoke", {}).get("failed_count", 0),
},
"permissions": {
"enforcement": model["permission_enforcement"],
"native_enforcement": bool(target_permissions.get("native_enforcement")),
"declared_capabilities": target_permissions.get("declared_capabilities", []),
"review_required": bool(target_permissions.get("review_required")),
},
"review": {
"artifacts": model["review_artifacts"],
"fallback_behavior": model["fallback_behavior"],
"unsupported_native_features": [
*model.get("unsupported_native_features", []),
*profile.get("unsupported_features", []),
],
},
"install_scope": model["install_scope"],
}
def load_sources(skill_dir: Path) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
frontmatter = read_frontmatter(skill_dir / "SKILL.md")
manifest = load_json(skill_dir / "manifest.json")
interface_doc = load_yaml(skill_dir / "agents" / "interface.yaml")
name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name)
ir, ir_source = find_skill_ir(skill_dir, name)
return {
"skill_dir": skill_dir,
"frontmatter": frontmatter,
"manifest": manifest,
"interface_doc": interface_doc,
"interface": interface_doc.get("interface", {}),
"compatibility": interface_doc.get("compatibility", {}),
"ir": ir,
"ir_source": ir_source,
}
def declared_targets(sources: dict[str, Any]) -> list[str]:
ir = sources["ir"]
compatibility = sources["compatibility"]
manifest = sources["manifest"]
candidates = (
ir.get("targets")
if isinstance(ir.get("targets"), list)
else manifest.get("target_platforms")
or compatibility.get("adapter_targets")
or []
)
targets = []
for item in candidates:
value = str(item).strip()
if value and value not in targets:
targets.append(value)
return targets or ["generic"]
def semantic_source(sources: dict[str, Any]) -> dict[str, Any]:
skill_dir = sources["skill_dir"]
frontmatter = sources["frontmatter"]
manifest = sources["manifest"]
interface = sources["interface"]
compatibility = sources["compatibility"]
ir = sources["ir"]
trigger = ir.get("trigger_surface", {}) if isinstance(ir.get("trigger_surface"), dict) else {}
workflow = ir.get("workflow", {}) if isinstance(ir.get("workflow"), dict) else {}
resources = ir.get("resources", {}) if isinstance(ir.get("resources"), dict) else {}
eval_plan = ir.get("eval_plan", {}) if isinstance(ir.get("eval_plan"), dict) else {}
description = str(trigger.get("description") or frontmatter.get("description") or "")
name = str(ir.get("name") or frontmatter.get("name") or manifest.get("name") or skill_dir.name)
permissions = permission_contract(skill_dir)
return {
"name": name,
"title": str(ir.get("title") or interface.get("display_name") or name),
"version": str(manifest.get("version") or frontmatter.get("version") or "1.0.0"),
"description": description,
"job_to_be_done": str(ir.get("job_to_be_done") or description),
"trigger": {
"description": description,
"should_trigger": list_or_empty(trigger, "should_trigger"),
"should_not_trigger": list_or_empty(trigger, "should_not_trigger"),
"edge_cases": list_or_empty(trigger, "edge_cases"),
},
"workflow": {
"steps": list_or_empty(workflow, "steps"),
"decision_points": list_or_empty(workflow, "decision_points"),
"failure_modes": list_or_empty(workflow, "failure_modes"),
},
"resources": {
"references": list_or_empty(resources, "references"),
"scripts": list_or_empty(resources, "scripts"),
"assets": list_or_empty(resources, "assets"),
"reports": list_or_empty(resources, "reports"),
"counts": resource_counts(resources),
},
"eval_plan": {
"trigger": list_or_empty(eval_plan, "trigger"),
"output": list_or_empty(eval_plan, "output"),
"adversarial": list_or_empty(eval_plan, "adversarial"),
"baseline": str(eval_plan.get("baseline") or ""),
"counts": eval_counts(eval_plan),
},
"risk": ir.get("risk", {}) if isinstance(ir.get("risk"), dict) else {},
"governance": ir.get("governance", {}) if isinstance(ir.get("governance"), dict) else {},
"runtime": {
"activation": compatibility.get("activation", {}) if isinstance(compatibility.get("activation"), dict) else {},
"execution": compatibility.get("execution", {}) if isinstance(compatibility.get("execution"), dict) else {},
"trust": compatibility.get("trust", {}) if isinstance(compatibility.get("trust"), dict) else {},
"adapter_targets": compatibility.get("adapter_targets", []),
"canonical_format": compatibility.get("canonical_format", "agent-skills"),
},
"permissions": permissions,
"source_files_count": count_list(ir, "source_files") if ir else 0,
}
def compile_target_contract(skill_dir: Path, target: str) -> dict[str, Any]:
sources = load_sources(skill_dir)
compatibility = sources["compatibility"]
degradation = compatibility.get("degradation", {}) if isinstance(compatibility.get("degradation"), dict) else {}
declared = declared_targets(sources)
failures: list[str] = []
warnings: list[str] = []
profile = TARGET_TRANSFORMS.get(target)
if profile is None:
failures.append(f"Unsupported compiler target: {target}")
profile = {
"adapter_mode": "unsupported",
"generated_files": [],
"metadata_mapping": {},
"preserved_semantics": [],
"unsupported_features": [],
}
if target not in declared and not (target == "agent-skills" and "agent-skills-compatible" in declared):
warnings.append(f"Target is not declared in Skill IR or interface metadata: {target}")
if not sources["ir"]:
warnings.append("Skill IR is missing; compiler used frontmatter fallback")
contract = semantic_source(sources)
permissions = contract["permissions"]
target_permissions = target_permission_contract(target, permissions)
target_native = target_native_contract(target, profile, contract, target_permissions)
contract["target"] = target
contract["target_permission_contract"] = target_permissions
contract["target_native_contract"] = target_native
contract["degradation_strategy"] = degradation.get(target, degradation.get("generic", "neutral-source"))
contract["target_runtime"] = {
"adapter_mode": profile["adapter_mode"],
"generated_files": profile["generated_files"],
"metadata_mapping": profile["metadata_mapping"],
"preserved_semantics": profile["preserved_semantics"],
"native_surface": target_native["native_surface"],
}
return {
"schema_version": COMPILER_SCHEMA_VERSION,
"target": target,
"status": "block" if failures else ("warn" if warnings else "pass"),
"compiler": {
"name": COMPILER_NAME,
"schema_version": COMPILER_SCHEMA_VERSION,
"source": "skill-ir" if sources["ir"] else "frontmatter-fallback",
"ir_source": sources["ir_source"],
"ir_schema_version": str(sources["ir"].get("schema_version") or "none"),
},
"source": {
"skill_dir": display_path(sources["skill_dir"]),
"canonical_metadata": "agents/interface.yaml" if (sources["skill_dir"] / "agents" / "interface.yaml").exists() else "missing",
"declared_targets": declared,
},
"compiled_contract": contract,
"permission_contract": permissions,
"target_permission_contract": target_permissions,
"target_native_contract": target_native,
"target_transform": {
"target": target,
"adapter_mode": profile["adapter_mode"],
"generated_files": profile["generated_files"],
"metadata_mapping": profile["metadata_mapping"],
"preserved_semantics": profile["preserved_semantics"],
"degradation_strategy": contract["degradation_strategy"],
"permission_representation": target_permissions["representation"],
"native_surface": target_native["native_surface"],
"activation_policy": target_native["activation"]["policy"],
"resource_strategy": target_native["resources"]["strategy"],
"script_strategy": target_native["scripts"]["strategy"],
"permission_enforcement": target_native["permissions"]["enforcement"],
},
"unsupported_features": list(profile.get("unsupported_features", [])),
"warnings": warnings,
"failures": failures,
}
def compile_targets(skill_dir: Path, targets: list[str] | None = None, generated_at: str | None = None) -> dict[str, Any]:
sources = load_sources(skill_dir)
selected = targets or declared_targets(sources)
compiled = [compile_target_contract(skill_dir, target) for target in selected]
failures = [failure for item in compiled for failure in item["failures"]]
warnings = [warning for item in compiled for warning in item["warnings"]]
return {
"schema_version": COMPILER_SCHEMA_VERSION,
"ok": not failures,
"generated_at": generated_at or str(date.today()),
"skill_dir": display_path(Path(skill_dir).resolve()),
"summary": {
"target_count": len(compiled),
"pass_count": sum(1 for item in compiled if item["status"] == "pass"),
"warn_count": sum(1 for item in compiled if item["status"] == "warn"),
"block_count": sum(1 for item in compiled if item["status"] == "block"),
"failure_count": len(failures),
"warning_count": len(warnings),
},
"targets": compiled,
"failures": failures,
"warnings": warnings,
}
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
"# Compiled Targets",
"",
f"- OK: `{report['ok']}`",
f"- Targets: `{summary['target_count']}`",
f"- Pass: `{summary['pass_count']}`",
f"- Warn: `{summary['warn_count']}`",
f"- Block: `{summary['block_count']}`",
"",
"## Target Transforms",
"",
"| Target | Status | Native Surface | Adapter Mode | Permissions | Degradation | Generated Files |",
"| --- | --- | --- | --- | --- | --- | --- |",
]
for item in report["targets"]:
transform = item["target_transform"]
files = ", ".join(transform.get("generated_files", [])) or "none"
permissions = ", ".join(item.get("target_permission_contract", {}).get("declared_capabilities", [])) or "none"
lines.append(
f"| `{item['target']}` | `{item['status']}` | {transform.get('native_surface', '')} | `{transform.get('adapter_mode', '')}` | `{permissions}` | `{transform.get('degradation_strategy', '')}` | {files} |"
)
lines.extend(["", "## Native Behavior Contracts", ""])
for item in report["targets"]:
native = item.get("target_native_contract", {})
permissions = native.get("permissions", {})
review = native.get("review", {})
lines.extend(
[
f"### {item['target']}",
"",
f"- Native surface: {native.get('native_surface', '')}",
f"- Activation: {native.get('activation', {}).get('policy', '')}",
f"- Resources: {native.get('resources', {}).get('strategy', '')}",
f"- Scripts: {native.get('scripts', {}).get('strategy', '')}",
f"- Permission enforcement: `{permissions.get('enforcement', '')}`; native enforcement `{permissions.get('native_enforcement')}`",
f"- Review artifacts: {', '.join(review.get('artifacts', [])) or 'none'}",
"",
]
)
lines.extend(["", "## Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
lines.extend(["", "## Warnings", ""])
lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"])
return "\n".join(lines).strip() + "\n"
def render_compile_report(
skill_dir: Path,
targets: list[str] | None = None,
output_json: Path | None = None,
output_md: Path | None = None,
generated_at: str | None = None,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
reports = skill_dir / "reports"
reports.mkdir(parents=True, exist_ok=True)
output_json = output_json or reports / "compiled_targets.json"
output_md = output_md or reports / "compiled_targets.md"
report = compile_targets(skill_dir, targets=targets, generated_at=generated_at)
report["artifacts"] = {
"json": display_path(output_json),
"markdown": display_path(output_md),
}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
return report
def main() -> None:
parser = argparse.ArgumentParser(description="Compile Skill IR into target-specific semantic contracts.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--target", action="append", default=[])
parser.add_argument("--output-json")
parser.add_argument("--output-md")
parser.add_argument("--generated-at")
args = parser.parse_args()
payload = render_compile_report(
Path(args.skill_dir),
targets=args.target or None,
output_json=Path(args.output_json).resolve() if args.output_json else None,
output_md=Path(args.output_md).resolve() if args.output_md else None,
generated_at=args.generated_at,
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 2)
if __name__ == "__main__":
main()
scripts/cross_packager.py
+59 -2
View File
@@ -6,6 +6,8 @@ import zipfile
from pathlib import Path
import yaml
from compile_skill import compile_target_contract
def display_path(path: Path, root: Path) -> str:
try:
@@ -199,10 +201,13 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
ir=ir,
ir_source=ir_source,
)
compiled = compile_target_contract(skill_dir, platform)
if compiled.get("failures"):
raise ValueError(f"Compiler failed for {platform}: {'; '.join(compiled['failures'])}")
return {
"name": semantic["name"],
"description": semantic["description"],
"version": frontmatter.get("version", "1.0.0"),
"version": manifest.get("version") or frontmatter.get("version", "1.0.0"),
"platform": platform,
"skill_root": skill_dir.name,
"job_to_be_done": semantic["job_to_be_done"],
@@ -210,6 +215,14 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
"ir_schema_version": semantic["ir_schema_version"],
"semantic_contract": semantic["semantic_contract"],
"semantic_parity": semantic["semantic_parity"],
"compiler": compiled["compiler"],
"compiled_contract": compiled["compiled_contract"],
"permission_contract": compiled["permission_contract"],
"target_permission_contract": compiled["target_permission_contract"],
"target_native_contract": compiled["target_native_contract"],
"target_transform": compiled["target_transform"],
"unsupported_features": compiled["unsupported_features"],
"compiler_warnings": compiled["warnings"],
"display_name": interface.get("display_name", skill_dir.name),
"short_description": interface.get("short_description", ""),
"default_prompt": interface.get("default_prompt", ""),
@@ -260,6 +273,9 @@ PLATFORM_CONTRACTS = {
"remote_inline_execution",
"degradation_strategy",
"portability_profile",
"permission_contract",
"target_permission_contract",
"target_native_contract",
],
"required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
"field_mapping": {
@@ -292,6 +308,9 @@ PLATFORM_CONTRACTS = {
"remote_inline_execution",
"degradation_strategy",
"portability_profile",
"permission_contract",
"target_permission_contract",
"target_native_contract",
],
"required_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
"field_mapping": {
@@ -324,6 +343,9 @@ PLATFORM_CONTRACTS = {
"remote_inline_execution",
"degradation_strategy",
"portability_profile",
"permission_contract",
"target_permission_contract",
"target_native_contract",
],
"required_files": ["targets/generic/adapter.json"],
"field_mapping": {
@@ -336,6 +358,19 @@ PLATFORM_CONTRACTS = {
},
}
EXCLUDED_ARCHIVE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"}
def should_skip_archive_path(rel_path: Path) -> bool:
parts = rel_path.parts
if any(part in EXCLUDED_ARCHIVE_PARTS for part in parts):
return True
if parts == ("reports", "telemetry_events.jsonl"):
return True
if parts and parts[0] == "tests" and any(part.startswith("tmp") for part in parts[1:]):
return True
return False
def is_relative_to(path: Path, parent: Path) -> bool:
try:
@@ -397,6 +432,19 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
"shell": payload["shell"],
"trust_level": payload["trust_level"],
"remote_inline_execution": payload["remote_inline_execution"],
"permission_contract": {
"review_required": payload["target_permission_contract"]["review_required"],
"declared_capabilities": payload["target_permission_contract"]["declared_capabilities"],
"native_enforcement": payload["target_permission_contract"]["native_enforcement"],
"representation": payload["target_permission_contract"]["representation"],
},
"native_contract": {
"native_surface": payload["target_native_contract"]["native_surface"],
"activation_policy": payload["target_native_contract"]["activation"]["policy"],
"resource_strategy": payload["target_native_contract"]["resources"]["strategy"],
"permission_enforcement": payload["target_native_contract"]["permissions"]["enforcement"],
"review_artifacts": payload["target_native_contract"]["review"]["artifacts"],
},
"degradation_strategy": payload["degradation_strategy"],
},
},
@@ -404,8 +452,14 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
payload["install_hint"] = f"Use the packaged skill and include targets/openai/agents/openai.yaml when the client expects OpenAI-style interface metadata."
elif platform == "claude":
notes = target_dir / "README.md"
native = payload["target_native_contract"]
notes.write_text(
f"# Claude-Compatible Package\n\nUse `{skill_dir.name}` with its neutral source files. This target does not require vendor metadata by default.\n",
f"# Claude-Compatible Package\n\nUse `{skill_dir.name}` with its neutral source files. This target does not require vendor metadata by default.\n\n"
f"Native surface: {native['native_surface']}.\n\n"
f"Activation: {native['activation']['policy']}\n\n"
f"Resources: {native['resources']['strategy']}\n\n"
f"Scripts: {native['scripts']['strategy']}\n\n"
f"Permission metadata is preserved in `adapter.json` under `target_permission_contract` and `target_native_contract` for reviewer visibility.\n",
encoding="utf-8",
)
payload["install_hint"] = f"Use the packaged skill directly; this target relies on SKILL.md and optional neutral metadata."
@@ -430,6 +484,9 @@ def make_zip(skill_dir: Path, out_dir: Path) -> Path:
continue
if is_relative_to(resolved, out_root):
continue
rel_path = path.relative_to(skill_dir)
if should_skip_archive_path(rel_path):
continue
zf.write(path, arcname=str(path.relative_to(skill_dir.parent)))
return zip_path
scripts/export_skill_ir.py
+11
View File
@@ -36,10 +36,21 @@ KEY_REPORTS = [
"reports/skill-overview.html",
"reports/output_quality_scorecard.json",
"reports/output_quality_scorecard.md",
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/output_blind_review_pack.json",
"reports/output_blind_review_pack.md",
"reports/output_blind_answer_key.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md",
"reports/review_annotations.json",
"reports/review_annotations.md",
"reports/conformance_matrix.json",
"reports/conformance_matrix.md",
"reports/security_trust_report.json",
"reports/security_trust_report.md",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"reports/skill_atlas.json",
"reports/skill_atlas.html",
"reports/skill-os-2-review.md",
scripts/init_skill.py
+35 -8
View File
@@ -5,11 +5,15 @@ import re
from datetime import date
from pathlib import Path
from compile_skill import render_compile_report
from export_skill_ir import build_skill_ir, validate_ir
from github_benchmark_scan import run_github_benchmark_scan
from render_intent_confidence import render_intent_confidence
from render_intent_dialogue import render_intent_dialogue
from render_iteration_directions import render_iteration_directions
from render_adoption_drift_report import render_report as render_adoption_drift_report
from render_review_annotations import render_report as render_review_annotations
from render_review_waivers import render_report as render_review_waivers
from render_artifact_design_profile import render_artifact_design_profile
from render_output_risk_profile import render_output_risk_profile
from render_prompt_quality_profile import render_prompt_quality_profile
@@ -71,13 +75,17 @@ README_TEMPLATE = """# {title}
6. Follow the workflow steps in `SKILL.md`.
7. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. It defaults to Simplified Chinese and includes an English switch in the top right.
8. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view.
9. Open `reports/review-viewer.html` for a compact visual review of the package.
10. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks.
11. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns.
12. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix.
13. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging.
14. Review `reports/iteration-directions.md` for the three most valuable next moves.
15. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes.
9. Record source-line reviewer comments in `reports/review_annotations.md` when review needs follow-up.
10. Open `reports/review-viewer.html` for a compact visual review of the package.
11. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks.
12. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns.
13. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix.
14. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging.
15. Review `reports/compiled_targets.md` to see how Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts.
16. Review `reports/iteration-directions.md` for the three most valuable next moves.
17. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes.
18. Review `reports/adoption_drift_report.md` to see local-first metadata-only adoption and drift signals.
19. Review `reports/review_waivers.md` to see human reviewer risk approvals and expiry dates.
## Honest Boundaries
@@ -100,10 +108,14 @@ README_TEMPLATE = """# {title}
- `reports/prompt-quality-profile.md`: prompt-facing need model, RTF mapping, complexity, and quality matrix
- `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points
- `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance
- `reports/compiled_targets.md`: target compiler report showing generated contracts, adapter modes, preserved semantics, warnings, and unsupported features
- `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap
- `reports/review-studio.html`: Review Studio 2.0 gate page for intent, trigger, output eval, context, runtime conformance, trust, atlas, and release readiness
- `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps
- `reports/iteration-directions.md`: the top three next iteration directions
- `reports/adoption_drift_report.md`: local-first metadata-only telemetry summary for adoption, missed triggers, bad outputs, script errors, and review drift
- `reports/review_waivers.md`: human reviewer risk approval ledger for warning acceptance and expiry
- `reports/review_annotations.md`: source-line reviewer comments linked to Review Studio gates
"""
@@ -223,8 +235,10 @@ def build_report_view(artifacts: dict) -> dict:
"review_studio": review_studio,
"message": (
f"Skill 已创建完成。建议先打开总结报告：{html_report}。"
"它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线；"
"它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线；"
f"然后打开 Review Studio 2.0：{review_studio}，检查意图、触发、输出评测、运行一致性、信任和发布闸门。"
"后续 reviewer 的文件级或行级意见可以记录到 reports/review_annotations.md。"
"如需审查平台适配细节，请打开 reports/compiled_targets.md。"
"报告默认使用中文简体，右上角可以切换英文版。"
),
"next_action": "Open reports/skill-overview.html before editing more files.",
@@ -336,7 +350,11 @@ def initialize_skill(
prompt_quality_profile = render_prompt_quality_profile(root)
system_model = render_system_model(root)
skill_ir = render_skill_ir(root)
compiled_targets = render_compile_report(root)
iteration_directions = render_iteration_directions(root)
adoption_drift = render_adoption_drift_report(root)
review_waivers = render_review_waivers(root)
review_annotations = render_review_annotations(root)
overview = render_skill_overview(root)
review_viewer = render_review_viewer(root)
review_studio = render_review_studio(root)
@@ -363,8 +381,16 @@ def initialize_skill(
"system_model_md": system_model["artifacts"]["markdown"],
"system_model_json": system_model["artifacts"]["json"],
"skill_ir_json": skill_ir["artifacts"]["json"],
"compiled_targets_md": compiled_targets["artifacts"]["markdown"],
"compiled_targets_json": compiled_targets["artifacts"]["json"],
"iteration_directions_md": iteration_directions["artifacts"]["markdown"],
"iteration_directions_json": iteration_directions["artifacts"]["json"],
"adoption_drift_md": adoption_drift["artifacts"]["markdown"],
"adoption_drift_json": adoption_drift["artifacts"]["json"],
"review_waivers_md": review_waivers["artifacts"]["markdown"],
"review_waivers_json": review_waivers["artifacts"]["json"],
"review_annotations_md": review_annotations["artifacts"]["markdown"],
"review_annotations_json": review_annotations["artifacts"]["json"],
"review_viewer_html": review_viewer["artifacts"]["html"],
"review_viewer_json": review_viewer["artifacts"]["json"],
"review_studio_html": absolute_skill_artifact(root, review_studio["artifacts"]["html"]),
@@ -384,6 +410,7 @@ def initialize_skill(
"reference_synthesis": reference_synthesis["summary"],
"system_model": system_model["summary"],
"skill_ir": skill_ir["summary"],
"compiled_targets": compiled_targets["summary"],
"artifacts": artifacts,
"report_view": report_view,
}
scripts/local_output_eval_runner.py
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env python3
import argparse
import json
import sys
from typing import Any
def estimate_tokens(text: str) -> int:
if not text:
return 0
return max(1, round(len(text) / 4))
def build_response(request: dict[str, Any], execution_kind: str, provider: str, model: str) -> dict[str, Any]:
output = str(request.get("fixture_output", ""))
prompt = str(request.get("prompt", ""))
input_tokens = estimate_tokens(prompt)
output_tokens = estimate_tokens(output)
response: dict[str, Any] = {
"output": output,
"execution_kind": execution_kind,
"provider": provider,
"usage": {
"input_tokens": input_tokens,
"output_tokens": output_tokens,
"total_tokens": input_tokens + output_tokens,
"estimated": True,
},
}
if model:
response["model"] = model
return response
def main() -> None:
parser = argparse.ArgumentParser(
description=(
"Local deterministic output-eval runner. It executes the runner contract "
"without claiming provider-backed model generation."
)
)
parser.add_argument("--execution-kind", choices=["command", "model"], default="command")
parser.add_argument("--provider", default="local-output-eval-runner")
parser.add_argument("--model", default="")
args = parser.parse_args()
raw = sys.stdin.read()
if not raw.strip():
print("runner requires a JSON request on stdin", file=sys.stderr)
raise SystemExit(2)
try:
request = json.loads(raw)
except json.JSONDecodeError as exc:
print(f"invalid JSON request: {exc}", file=sys.stderr)
raise SystemExit(2)
if not isinstance(request, dict):
print("runner request must be a JSON object", file=sys.stderr)
raise SystemExit(2)
response = build_response(request, args.execution_kind, args.provider, args.model)
print(json.dumps(response, ensure_ascii=False))
if __name__ == "__main__":
main()
scripts/probe_runtime_permissions.py
+289
View File
@@ -0,0 +1,289 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
from typing import Any
try:
import yaml
except ImportError: # pragma: no cover
yaml = None
DEFAULT_TARGETS = ["openai", "claude", "generic"]
def display_path(path: Path, root: Path) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def load_yaml(path: Path) -> dict[str, Any]:
if not path.exists() or yaml is None:
return {}
try:
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
except yaml.YAMLError:
return {}
return payload if isinstance(payload, dict) else {}
def expected_capabilities(skill_dir: Path) -> list[str]:
trust = load_json(skill_dir / "reports" / "security_trust_report.json")
governance = trust.get("permission_governance", {}) if isinstance(trust.get("permission_governance"), dict) else {}
required = governance.get("required_capabilities", [])
if isinstance(required, list) and required:
return sorted(str(item) for item in required)
summary = trust.get("summary", {}) if isinstance(trust.get("summary"), dict) else {}
candidates = []
if int(summary.get("network_script_count", 0) or 0):
candidates.append("network")
if int(summary.get("file_write_script_count", 0) or 0):
candidates.append("file_write")
if any(item.get("uses_subprocess") for item in trust.get("scripts", []) if isinstance(item, dict)):
candidates.append("subprocess")
if int(summary.get("interactive_script_count", 0) or 0):
candidates.append("interactive")
return sorted(candidates)
def add_check(checks: list[dict[str, Any]], failures: list[str], key: str, condition: bool, detail: str) -> None:
checks.append({"key": key, "passed": condition, "detail": detail})
if not condition:
failures.append(detail)
def sorted_strings(value: Any) -> list[str]:
return sorted(str(item) for item in value) if isinstance(value, list) else []
def probe_openai_yaml(package_dir: Path, expected: list[str]) -> tuple[list[dict[str, Any]], list[str]]:
checks: list[dict[str, Any]] = []
failures: list[str] = []
path = package_dir / "targets" / "openai" / "agents" / "openai.yaml"
payload = load_yaml(path)
permission_contract = payload.get("compatibility", {}).get("permission_contract", {}) if payload else {}
add_check(checks, failures, "openai-yaml-present", bool(payload), "OpenAI permission metadata YAML is readable")
add_check(
checks,
failures,
"openai-yaml-permissions",
sorted_strings(permission_contract.get("declared_capabilities")) == expected,
"OpenAI YAML permission contract mirrors expected capabilities",
)
add_check(
checks,
failures,
"openai-yaml-native-flag",
isinstance(permission_contract.get("native_enforcement"), bool),
"OpenAI YAML declares native_enforcement as a boolean",
)
return checks, failures
def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list[str]) -> dict[str, Any]:
adapter_path = package_dir / "targets" / target / "adapter.json"
checks: list[dict[str, Any]] = []
failures: list[str] = []
adapter = load_json(adapter_path)
add_check(checks, failures, "adapter-present", bool(adapter), f"{target} adapter.json is readable")
permission_contract = adapter.get("permission_contract", {}) if adapter else {}
target_contract = adapter.get("target_permission_contract", {}) if adapter else {}
add_check(checks, failures, "permission-contract-present", bool(permission_contract), f"{target} adapter includes permission_contract")
add_check(checks, failures, "target-contract-present", bool(target_contract), f"{target} adapter includes target_permission_contract")
add_check(
checks,
failures,
"source-available",
permission_contract.get("source_available") is True,
f"{target} permission_contract links to an available trust report",
)
add_check(
checks,
failures,
"declared-capabilities-match",
sorted_strings(target_contract.get("declared_capabilities")) == expected,
f"{target} target_permission_contract mirrors expected capabilities",
)
add_check(
checks,
failures,
"capability-counts-present",
isinstance(target_contract.get("capability_counts"), dict),
f"{target} target_permission_contract includes capability_counts",
)
add_check(
checks,
failures,
"native-enforcement-boolean",
isinstance(target_contract.get("native_enforcement"), bool),
f"{target} target_permission_contract declares native_enforcement as a boolean",
)
add_check(
checks,
failures,
"representation-present",
bool(str(target_contract.get("representation", "")).strip()),
f"{target} target_permission_contract declares where permission metadata is represented",
)
add_check(
checks,
failures,
"operator-note-present",
bool(str(target_contract.get("operator_note", "")).strip()),
f"{target} target_permission_contract includes an operator_note",
)
add_check(
checks,
failures,
"review-required-matches",
bool(target_contract.get("review_required")) == bool(expected),
f"{target} review_required matches whether capabilities are required",
)
yaml_checks: list[dict[str, Any]] = []
yaml_failures: list[str] = []
if target == "openai":
yaml_checks, yaml_failures = probe_openai_yaml(package_dir, expected)
checks.extend(yaml_checks)
failures.extend(yaml_failures)
native = target_contract.get("native_enforcement")
metadata_fallback = native is False and bool(target_contract.get("representation")) and bool(target_contract.get("operator_note"))
assurance = "native-enforced" if native is True else ("metadata-fallback-explicit" if metadata_fallback else "missing")
residual_risks = []
if native is False:
residual_risks.append("Client-native permission enforcement is not provided by this target; installer or operator must honor metadata.")
return {
"target": target,
"status": "pass" if not failures else "fail",
"adapter": display_path(adapter_path, skill_dir),
"permission_model": str(target_contract.get("permission_model", "")),
"native_enforcement": bool(native) if isinstance(native, bool) else None,
"metadata_fallback_explicit": metadata_fallback,
"assurance": assurance,
"declared_capabilities": sorted_strings(target_contract.get("declared_capabilities")),
"checks": checks,
"failures": failures,
"residual_risks": residual_risks,
}
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
"# Runtime Permission Probes",
"",
"Runtime permission probes verify that generated target adapters expose high-permission capabilities and make native-enforcement limits explicit.",
"",
"## Summary",
"",
f"- OK: `{report['ok']}`",
f"- Targets probed: `{summary['target_count']}`",
f"- Passed: `{summary['pass_count']}`",
f"- Failed: `{summary['fail_count']}`",
f"- Native enforcement targets: `{summary['native_enforcement_count']}`",
f"- Explicit metadata fallbacks: `{summary['metadata_fallback_count']}`",
f"- Required capabilities: `{', '.join(report['expected_capabilities']) or 'none'}`",
"",
"| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Residual Risk |",
"| --- | --- | --- | --- | --- | --- |",
]
for target in report["targets"]:
residual = "<br>".join(target["residual_risks"]) if target["residual_risks"] else "None"
lines.append(
f"| `{target['target']}` | `{target['status']}` | `{target['assurance']}` | "
f"`{target['native_enforcement']}` | `{target['metadata_fallback_explicit']}` | {residual} |"
)
lines.extend(["", "## Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
lines.extend(
[
"",
"## Reviewer Note",
"",
"A passing probe means the target contract is explicit and auditable. It does not claim that a host client enforces permissions natively.",
]
)
return "\n".join(lines).strip() + "\n"
def probe_runtime_permissions(
skill_dir: Path,
package_dir: Path,
targets: list[str],
output_json: Path,
output_md: Path,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
package_dir = package_dir.resolve()
expected = expected_capabilities(skill_dir)
target_results = [probe_target(skill_dir, package_dir, target, expected) for target in targets]
failures = [failure for target in target_results for failure in target["failures"]]
summary = {
"target_count": len(target_results),
"pass_count": sum(1 for item in target_results if item["status"] == "pass"),
"fail_count": sum(1 for item in target_results if item["status"] == "fail"),
"native_enforcement_count": sum(1 for item in target_results if item["native_enforcement"] is True),
"metadata_fallback_count": sum(1 for item in target_results if item["metadata_fallback_explicit"]),
"residual_risk_count": sum(len(item["residual_risks"]) for item in target_results),
"required_capability_count": len(expected),
"failure_count": len(failures),
}
report = {
"schema_version": "1.0",
"ok": not failures,
"skill_dir": display_path(skill_dir, skill_dir),
"package_dir": display_path(package_dir, skill_dir),
"expected_capabilities": expected,
"summary": summary,
"targets": target_results,
"failures": failures,
"artifacts": {
"json": display_path(output_json, skill_dir),
"markdown": display_path(output_md, skill_dir),
},
}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
return report
def main() -> None:
parser = argparse.ArgumentParser(description="Probe generated target adapters for runtime permission enforcement metadata.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--package-dir", default="dist")
parser.add_argument("--target", action="append", choices=DEFAULT_TARGETS)
parser.add_argument("--output-json")
parser.add_argument("--output-md")
args = parser.parse_args()
skill_dir = Path(args.skill_dir).resolve()
report = probe_runtime_permissions(
skill_dir,
Path(args.package_dir).resolve(),
args.target or DEFAULT_TARGETS,
Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "runtime_permission_probes.json",
Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "runtime_permission_probes.md",
)
print(json.dumps(report, ensure_ascii=False, indent=2))
raise SystemExit(0 if report["ok"] else 2)
if __name__ == "__main__":
main()
scripts/registry_audit.py
+377
View File
@@ -0,0 +1,377 @@
#!/usr/bin/env python3
import argparse
import json
import re
from datetime import date
from pathlib import Path
from typing import Any
try:
import yaml
except ImportError: # pragma: no cover
yaml = None
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_REGISTRY_DIR = ROOT / "registry"
REQUIRED_PACKAGE_FIELDS = [
"name",
"version",
"description",
"targets",
"maturity",
"owner",
"review_cadence",
"trust_level",
"license",
]
def display_path(path: Path, root: Path = ROOT) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def load_yaml(path: Path) -> dict[str, Any]:
if not path.exists() or yaml is None:
return {}
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
return payload if isinstance(payload, dict) else {}
def read_frontmatter(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
text = path.read_text(encoding="utf-8", errors="replace")
if not text.startswith("---"):
return {}
parts = text.split("---", 2)
if len(parts) < 3:
return {}
if yaml is not None:
payload = yaml.safe_load(parts[1]) or {}
return payload if isinstance(payload, dict) else {}
data: dict[str, Any] = {}
for line in parts[1].splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
data[key.strip()] = value.strip().strip('"')
return data
def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]:
candidates = [
skill_dir / "reports" / "skill-ir.json",
skill_dir / "skill-ir" / "examples" / f"{name}.json",
skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json",
]
for path in candidates:
payload = load_json(path)
if payload:
return payload, display_path(path, skill_dir)
return {}, "missing"
def license_id(skill_dir: Path) -> str:
for name in ("LICENSE", "LICENSE.md", "LICENSE.txt"):
path = skill_dir / name
if not path.exists():
continue
text = path.read_text(encoding="utf-8", errors="replace")
if re.search(r"\bMIT License\b", text, re.IGNORECASE):
return "MIT"
for line in text.splitlines():
value = line.strip()
if value:
return value[:80]
return ""
def compatibility_matrix(skill_dir: Path, declared_targets: list[str]) -> dict[str, str]:
conformance = load_json(skill_dir / "reports" / "conformance_matrix.json")
matrix = {}
for target in conformance.get("targets", []):
name = str(target.get("target", ""))
if name:
matrix[name] = str(target.get("status", "unknown"))
if "agent-skills-compatible" in declared_targets and "agent-skills" in matrix:
matrix["agent-skills-compatible"] = matrix["agent-skills"]
for target in declared_targets:
matrix.setdefault(str(target), "missing")
return matrix
def package_hash(skill_dir: Path) -> str:
trust = load_json(skill_dir / "reports" / "security_trust_report.json")
return str(trust.get("summary", {}).get("package_sha256", ""))
def package_verification(skill_dir: Path) -> dict[str, Any]:
return load_json(skill_dir / "reports" / "package_verification.json")
def install_simulation(skill_dir: Path) -> dict[str, Any]:
return load_json(skill_dir / "reports" / "install_simulation.json")
def adoption_drift(skill_dir: Path) -> dict[str, Any]:
return load_json(skill_dir / "reports" / "adoption_drift_report.json")
def review_waivers(skill_dir: Path) -> dict[str, Any]:
return load_json(skill_dir / "reports" / "review_waivers.json")
def review_annotations(skill_dir: Path) -> dict[str, Any]:
return load_json(skill_dir / "reports" / "review_annotations.json")
def compiled_targets(skill_dir: Path) -> dict[str, Any]:
return load_json(skill_dir / "reports" / "compiled_targets.json")
def package_targets(manifest: dict[str, Any], interface_doc: dict[str, Any], ir: dict[str, Any]) -> list[str]:
candidates = (
manifest.get("target_platforms")
or ir.get("targets")
or interface_doc.get("compatibility", {}).get("adapter_targets")
or []
)
targets = []
for item in candidates:
value = str(item).strip()
if value and value not in targets:
targets.append(value)
return targets
def build_package_metadata(skill_dir: Path, generated_at: str) -> dict[str, Any]:
frontmatter = read_frontmatter(skill_dir / "SKILL.md")
manifest = load_json(skill_dir / "manifest.json")
interface_doc = load_yaml(skill_dir / "agents" / "interface.yaml")
compatibility = interface_doc.get("compatibility", {})
trust = compatibility.get("trust", {})
name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name)
ir, ir_source = find_skill_ir(skill_dir, name)
verification = package_verification(skill_dir)
install = install_simulation(skill_dir)
adoption = adoption_drift(skill_dir)
waivers = review_waivers(skill_dir)
annotations = review_annotations(skill_dir)
compiled = compiled_targets(skill_dir)
archive_sha = str(verification.get("summary", {}).get("archive_sha256", ""))
checksums = {"package_sha256": package_hash(skill_dir)}
if archive_sha:
checksums["archive_sha256"] = archive_sha
description = str(
ir.get("trigger_surface", {}).get("description")
or frontmatter.get("description")
or manifest.get("description")
or ""
)
targets = package_targets(manifest, interface_doc, ir)
package = {
"schema_version": "2.0",
"name": name,
"version": str(manifest.get("version") or frontmatter.get("version") or ""),
"description": description,
"targets": targets,
"maturity": str(manifest.get("maturity_tier") or manifest.get("lifecycle_stage") or "scaffold"),
"owner": str(manifest.get("owner") or ""),
"review_cadence": str(manifest.get("review_cadence") or ""),
"trust_level": str(trust.get("source_tier") or ""),
"license": license_id(skill_dir),
"checksums": checksums,
"compatibility": compatibility_matrix(skill_dir, targets),
"source": {
"skill_dir": display_path(skill_dir),
"skill_ir": ir_source,
"ir_schema_version": str(ir.get("schema_version") or "missing"),
"canonical_metadata": "agents/interface.yaml" if (skill_dir / "agents" / "interface.yaml").exists() else "missing",
},
"artifacts": {
"overview_html": "reports/skill-overview.html",
"review_studio_html": "reports/review-studio.html",
"trust_report": "reports/security_trust_report.md",
"conformance_matrix": "reports/conformance_matrix.md",
"compiled_targets": "reports/compiled_targets.md" if compiled else "",
"package_verification": "reports/package_verification.md" if verification else "",
"install_simulation": "reports/install_simulation.md" if install else "",
"adoption_drift": "reports/adoption_drift_report.md" if adoption else "",
"review_waivers": "reports/review_waivers.md" if waivers else "",
"review_annotations": "reports/review_annotations.md" if annotations else "",
"package_metadata": f"registry/packages/{name}.json",
},
"distribution": {
"archive_verified": bool(verification.get("ok")) if verification else False,
"archive_sha256": archive_sha,
"package_verification": "reports/package_verification.json" if verification else "",
"install_simulated": bool(install.get("ok")) if install else False,
"install_simulation": "reports/install_simulation.json" if install else "",
},
"generated_at": generated_at,
}
return package
def audit_package(package: dict[str, Any], skill_dir: Path) -> tuple[list[str], list[str]]:
failures = []
warnings = []
for field in REQUIRED_PACKAGE_FIELDS:
if not package.get(field):
failures.append(f"Missing package metadata field: {field}")
if not package.get("checksums", {}).get("package_sha256"):
failures.append("Missing package checksum: checksums.package_sha256")
if package.get("source", {}).get("ir_schema_version") != "2.0.0":
failures.append("Missing valid Skill IR source for registry package")
compatibility = package.get("compatibility", {})
for target in package.get("targets", []):
if compatibility.get(target) not in {"pass", "warn"}:
failures.append(f"Compatibility is not passing for target: {target}")
for rel in (
"reports/skill-overview.html",
"reports/review-studio.html",
"reports/compiled_targets.md",
"reports/security_trust_report.md",
"reports/review_waivers.md",
"reports/review_annotations.md",
):
if not (skill_dir / rel).exists():
warnings.append(f"Recommended registry evidence is missing: {rel}")
verification = package.get("distribution", {})
if verification.get("package_verification") and not verification.get("archive_verified"):
failures.append("Package verification report exists but archive verification did not pass")
if verification.get("install_simulation") and not verification.get("install_simulated"):
failures.append("Install simulation report exists but install simulation did not pass")
return failures, warnings
def build_index(package: dict[str, Any], generated_at: str) -> dict[str, Any]:
return {
"schema_version": "2.0",
"generated_at": generated_at,
"package_count": 1,
"packages": [
{
"name": package["name"],
"version": package["version"],
"maturity": package["maturity"],
"owner": package["owner"],
"targets": package["targets"],
"package_metadata": package["artifacts"]["package_metadata"],
"package_sha256": package["checksums"]["package_sha256"],
}
],
}
def render_markdown(report: dict[str, Any]) -> str:
package = report["package"]
lines = [
"# Registry Audit",
"",
f"- OK: `{report['ok']}`",
f"- Package: `{package['name']}`",
f"- Version: `{package['version']}`",
f"- Maturity: `{package['maturity']}`",
f"- Owner: `{package['owner']}`",
f"- License: `{package['license']}`",
f"- Package SHA256: `{package['checksums']['package_sha256']}`",
f"- Archive SHA256: `{package['checksums'].get('archive_sha256', 'n/a')}`",
f"- Install simulated: `{package.get('distribution', {}).get('install_simulated', False)}`",
"",
"## Compatibility",
"",
"| Target | Status |",
"| --- | --- |",
]
for target, status in package["compatibility"].items():
lines.append(f"| `{target}` | `{status}` |")
lines.extend(["", "## Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
lines.extend(["", "## Warnings", ""])
lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"])
lines.extend(["", "## Artifacts", ""])
for key, value in report["artifacts"].items():
lines.append(f"- {key}: `{value}`")
return "\n".join(lines) + "\n"
def run_registry_audit(
skill_dir: Path,
registry_dir: Path,
output_json: Path,
output_md: Path,
generated_at: str,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
registry_dir = registry_dir.resolve()
package = build_package_metadata(skill_dir, generated_at)
failures, warnings = audit_package(package, skill_dir)
index = build_index(package, generated_at)
package_dir = registry_dir / "packages"
package_path = package_dir / f"{package['name']}.json"
index_path = registry_dir / "index.json"
package_dir.mkdir(parents=True, exist_ok=True)
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
package_path.write_text(json.dumps(package, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
index_path.write_text(json.dumps(index, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
report = {
"ok": not failures,
"schema_version": "2.0",
"skill_dir": display_path(skill_dir),
"registry_dir": display_path(registry_dir),
"package": package,
"index": index,
"failures": failures,
"warnings": warnings,
"artifacts": {
"index": display_path(index_path),
"package": display_path(package_path),
"json": display_path(output_json),
"markdown": display_path(output_md),
},
}
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
return report
def main() -> None:
parser = argparse.ArgumentParser(description="Build and audit Skill OS registry package metadata.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--registry-dir", default=str(DEFAULT_REGISTRY_DIR))
parser.add_argument("--output-json", default="reports/registry_audit.json")
parser.add_argument("--output-md", default="reports/registry_audit.md")
parser.add_argument("--generated-at", default=str(date.today()))
args = parser.parse_args()
payload = run_registry_audit(
Path(args.skill_dir),
Path(args.registry_dir),
Path(args.output_json),
Path(args.output_md),
args.generated_at,
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 2)
if __name__ == "__main__":
main()
scripts/render_adoption_drift_report.py
+441
View File
@@ -0,0 +1,441 @@
#!/usr/bin/env python3
import argparse
import json
from collections import Counter, defaultdict
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parent.parent
ALLOWED_EVENTS = {"skill_activation", "skill_output", "script_run", "review_event"}
ALLOWED_ACTIVATION_TYPES = {"implicit", "explicit", "manual", "unknown"}
ALLOWED_OUTCOMES = {"accepted", "edited", "rejected", "missed", "failed", "reviewed", "unknown"}
ALLOWED_FAILURE_TYPES = {
"none",
"wrong_trigger",
"under_trigger",
"bad_output",
"missing_resource",
"script_error",
"review_overdue",
}
ALLOWED_FIELDS = {
"event",
"skill",
"version",
"activation_type",
"outcome",
"failure_type",
"timestamp",
}
SENSITIVE_FIELDS = {
"prompt",
"content",
"input",
"inputs",
"output",
"outputs",
"transcript",
"message",
"messages",
"note",
"text",
"raw",
}
def display_path(path: Path, root: Path = ROOT) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def read_frontmatter(path: Path) -> dict[str, str]:
if not path.exists():
return {}
text = path.read_text(encoding="utf-8", errors="replace")
if not text.startswith("---"):
return {}
parts = text.split("---", 2)
if len(parts) < 3:
return {}
data: dict[str, str] = {}
for line in parts[1].splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
data[key.strip()] = value.strip().strip('"')
return data
def skill_defaults(skill_dir: Path) -> dict[str, str]:
manifest = load_json(skill_dir / "manifest.json")
frontmatter = read_frontmatter(skill_dir / "SKILL.md")
return {
"skill": str(manifest.get("name") or frontmatter.get("name") or skill_dir.name),
"version": str(manifest.get("version") or frontmatter.get("version") or "0.0.0"),
}
def utc_now() -> str:
return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z")
def normalize_event(raw: dict[str, Any], defaults: dict[str, str], line_label: str) -> tuple[dict[str, str] | None, list[str]]:
failures: list[str] = []
raw_keys = set(raw.keys())
sensitive = sorted(raw_keys & SENSITIVE_FIELDS)
if sensitive:
failures.append(f"{line_label}: raw content fields are not allowed in telemetry: {', '.join(sensitive)}")
unknown = sorted(raw_keys - ALLOWED_FIELDS - SENSITIVE_FIELDS)
if unknown:
failures.append(f"{line_label}: unknown telemetry fields are blocked by the metadata-only contract: {', '.join(unknown)}")
event = str(raw.get("event") or "skill_activation")
activation_type = str(raw.get("activation_type") or "unknown")
outcome = str(raw.get("outcome") or "unknown")
failure_type = str(raw.get("failure_type") or "none")
timestamp = str(raw.get("timestamp") or utc_now())
skill = str(raw.get("skill") or defaults["skill"])
version = str(raw.get("version") or defaults["version"])
if event not in ALLOWED_EVENTS:
failures.append(f"{line_label}: unsupported event `{event}`")
if activation_type not in ALLOWED_ACTIVATION_TYPES:
failures.append(f"{line_label}: unsupported activation_type `{activation_type}`")
if outcome not in ALLOWED_OUTCOMES:
failures.append(f"{line_label}: unsupported outcome `{outcome}`")
if failure_type not in ALLOWED_FAILURE_TYPES:
failures.append(f"{line_label}: unsupported failure_type `{failure_type}`")
if failures:
return None, failures
return {
"event": event,
"skill": skill,
"version": version,
"activation_type": activation_type,
"outcome": outcome,
"failure_type": failure_type,
"timestamp": timestamp,
}, []
def load_events(path: Path, defaults: dict[str, str]) -> tuple[list[dict[str, str]], list[str]]:
if not path.exists():
return [], []
events: list[dict[str, str]] = []
failures: list[str] = []
for index, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
if not line.strip():
continue
try:
raw = json.loads(line)
except json.JSONDecodeError as exc:
failures.append(f"line {index}: invalid JSONL event: {exc.msg}")
continue
if not isinstance(raw, dict):
failures.append(f"line {index}: telemetry event must be a JSON object")
continue
event, event_failures = normalize_event(raw, defaults, f"line {index}")
failures.extend(event_failures)
if event:
events.append(event)
return events, failures
def append_event(path: Path, event: dict[str, str]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("a", encoding="utf-8") as handle:
handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n")
def adoption_by_skill(events: list[dict[str, str]]) -> list[dict[str, Any]]:
grouped: dict[str, Counter[str]] = defaultdict(Counter)
for event in events:
grouped[event["skill"]][event["outcome"]] += 1
rows = []
for skill, counts in sorted(grouped.items()):
total = sum(counts.values())
adopted = counts["accepted"] + counts["edited"]
rows.append(
{
"skill": skill,
"events": total,
"accepted": counts["accepted"],
"edited": counts["edited"],
"rejected": counts["rejected"],
"missed": counts["missed"],
"adoption_rate": round(adopted / total * 100, 1) if total else 0,
}
)
return rows
def atlas_review_overdue(skill_dir: Path) -> int:
atlas = load_json(skill_dir / "reports" / "skill_atlas.json")
stale = atlas.get("stale_skills", []) if isinstance(atlas.get("stale_skills"), list) else []
return sum(
1
for item in stale
if item.get("actionable", True) and "overdue" in str(item.get("reason", "")).lower()
)
def next_candidates(summary: dict[str, Any]) -> list[dict[str, str]]:
candidates: list[dict[str, str]] = []
if summary["event_count"] == 0:
candidates.append(
{
"signal": "no telemetry",
"recommendation": "Start with a small metadata-only sample before using telemetry for release decisions.",
}
)
if summary["missed_trigger_count"]:
candidates.append(
{
"signal": "missed trigger",
"recommendation": "Add missed prompts to trigger eval and tighten should-trigger examples.",
}
)
if summary["wrong_trigger_count"]:
candidates.append(
{
"signal": "wrong trigger",
"recommendation": "Add near-neighbor should-not-trigger cases and clarify exclusions.",
}
)
if summary["bad_output_count"]:
candidates.append(
{
"signal": "bad output",
"recommendation": "Convert the failed output shape into an Output Eval assertion and failure taxonomy entry.",
}
)
if summary["script_error_count"]:
candidates.append(
{
"signal": "script error",
"recommendation": "Add non-interactive script smoke tests and clearer error paths.",
}
)
if summary["review_overdue_count"]:
candidates.append(
{
"signal": "review overdue",
"recommendation": "Prioritize stale governed skills in Skill Atlas before creating more assets.",
}
)
return candidates[:6]
def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[str, Any]:
outcomes = Counter(event["outcome"] for event in events)
failures = Counter(event["failure_type"] for event in events if event["failure_type"] != "none")
event_types = Counter(event["event"] for event in events)
adopted = outcomes["accepted"] + outcomes["edited"]
event_count = len(events)
missed_trigger = outcomes["missed"] + failures["under_trigger"]
bad_output = failures["bad_output"]
script_error = failures["script_error"]
wrong_trigger = failures["wrong_trigger"]
risk_band = "no-data"
if event_count:
risk_points = missed_trigger + bad_output + script_error + wrong_trigger + review_overdue_count
if risk_points >= 4:
risk_band = "high"
elif risk_points:
risk_band = "medium"
else:
risk_band = "low"
return {
"event_count": event_count,
"activation_count": event_types["skill_activation"],
"accepted_count": outcomes["accepted"],
"edited_count": outcomes["edited"],
"rejected_count": outcomes["rejected"],
"missed_count": outcomes["missed"],
"failed_count": outcomes["failed"],
"adoption_rate": round(adopted / event_count * 100, 1) if event_count else 0,
"missed_trigger_count": missed_trigger,
"wrong_trigger_count": wrong_trigger,
"bad_output_count": bad_output,
"script_error_count": script_error,
"missing_resource_count": failures["missing_resource"],
"review_overdue_count": review_overdue_count + failures["review_overdue"],
"risk_band": risk_band,
"event_types": dict(sorted(event_types.items())),
"failure_types": dict(sorted(failures.items())),
}
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
"# Adoption And Drift Report",
"",
"Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, transcripts, and notes are not allowed in the event stream.",
"",
"## Summary",
"",
f"- Events: `{summary['event_count']}`",
f"- Activation events: `{summary['activation_count']}`",
f"- Adoption rate: `{summary['adoption_rate']}`",
f"- Missed trigger signals: `{summary['missed_trigger_count']}`",
f"- Bad output signals: `{summary['bad_output_count']}`",
f"- Script error signals: `{summary['script_error_count']}`",
f"- Review overdue signals: `{summary['review_overdue_count']}`",
f"- Risk band: `{summary['risk_band']}`",
"",
"## Privacy Contract",
"",
"- Storage is local-first.",
"- Events are metadata-only.",
"- Raw user prompts, model outputs, transcripts, notes, and messages are blocked.",
"- Distributed packages should include this aggregate report, not raw `reports/telemetry_events.jsonl`.",
"",
"## Adoption By Skill",
"",
"| Skill | Events | Accepted | Edited | Rejected | Missed | Adoption Rate |",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: |",
]
for row in report["adoption_by_skill"]:
lines.append(
f"| `{row['skill']}` | {row['events']} | {row['accepted']} | {row['edited']} | "
f"{row['rejected']} | {row['missed']} | {row['adoption_rate']} |"
)
if not report["adoption_by_skill"]:
lines.append("| `none` | 0 | 0 | 0 | 0 | 0 | 0 |")
lines.extend(["", "## Next Iteration Candidates", ""])
for item in report["next_iteration_candidates"]:
lines.append(f"- `{item['signal']}`: {item['recommendation']}")
if not report["next_iteration_candidates"]:
lines.append("- No telemetry-driven iteration candidate yet.")
lines.extend(["", "## Recent Metadata Events", ""])
for event in report["recent_events"]:
lines.append(
f"- `{event['timestamp']}` `{event['skill']}` event=`{event['event']}` "
f"activation=`{event['activation_type']}` outcome=`{event['outcome']}` failure=`{event['failure_type']}`"
)
if not report["recent_events"]:
lines.append("- No metadata events captured yet.")
if report["failures"]:
lines.extend(["", "## Validation Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]])
return "\n".join(lines) + "\n"
def render_report(
skill_dir: Path,
events_jsonl: Path | None = None,
output_json: Path | None = None,
output_md: Path | None = None,
generated_at: str | None = None,
record_event: dict[str, Any] | None = None,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
reports_dir = skill_dir / "reports"
reports_dir.mkdir(parents=True, exist_ok=True)
defaults = skill_defaults(skill_dir)
events_jsonl = events_jsonl or reports_dir / "telemetry_events.jsonl"
output_json = output_json or reports_dir / "adoption_drift_report.json"
output_md = output_md or reports_dir / "adoption_drift_report.md"
generated_at = generated_at or utc_now()
failures: list[str] = []
if record_event is not None:
event, event_failures = normalize_event(record_event, defaults, "record")
failures.extend(event_failures)
if event:
append_event(events_jsonl, event)
events, load_failures = load_events(events_jsonl, defaults)
failures.extend(load_failures)
summary = summarize(events, atlas_review_overdue(skill_dir))
report = {
"ok": not failures,
"schema_version": "2.0",
"generated_at": generated_at,
"skill_dir": display_path(skill_dir),
"privacy_contract": {
"storage": "local-first",
"event_scope": "metadata-only",
"raw_content_allowed": False,
"raw_event_log_packaged": False,
"blocked_fields": sorted(SENSITIVE_FIELDS),
},
"summary": summary,
"adoption_by_skill": adoption_by_skill(events),
"next_iteration_candidates": next_candidates(summary),
"recent_events": events[-10:],
"failures": failures,
"artifacts": {
"events_jsonl": display_path(events_jsonl),
"json": display_path(output_json),
"markdown": display_path(output_md),
},
}
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
return report
def main() -> None:
parser = argparse.ArgumentParser(description="Render local-first adoption and drift telemetry for a skill package.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--events-jsonl")
parser.add_argument("--output-json")
parser.add_argument("--output-md")
parser.add_argument("--generated-at")
parser.add_argument("--record-event", choices=sorted(ALLOWED_EVENTS))
parser.add_argument("--activation-type", choices=sorted(ALLOWED_ACTIVATION_TYPES), default="unknown")
parser.add_argument("--outcome", choices=sorted(ALLOWED_OUTCOMES), default="unknown")
parser.add_argument("--failure-type", choices=sorted(ALLOWED_FAILURE_TYPES), default="none")
parser.add_argument("--timestamp")
parser.add_argument("--skill-name")
parser.add_argument("--version")
args = parser.parse_args()
record_event = None
if args.record_event:
record_event = {
"event": args.record_event,
"activation_type": args.activation_type,
"outcome": args.outcome,
"failure_type": args.failure_type,
}
if args.timestamp:
record_event["timestamp"] = args.timestamp
if args.skill_name:
record_event["skill"] = args.skill_name
if args.version:
record_event["version"] = args.version
report = render_report(
Path(args.skill_dir),
events_jsonl=Path(args.events_jsonl).resolve() if args.events_jsonl else None,
output_json=Path(args.output_json).resolve() if args.output_json else None,
output_md=Path(args.output_md).resolve() if args.output_md else None,
generated_at=args.generated_at,
record_event=record_event,
)
print(json.dumps(report, ensure_ascii=False, indent=2))
if not report["ok"]:
raise SystemExit(2)
if __name__ == "__main__":
main()
scripts/render_context_reports.py
+13 -1
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env python3
import argparse
import json
from pathlib import Path
@@ -25,7 +26,18 @@ TARGETS = [
]
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Render context budget reports for root and example skills.")
parser.add_argument(
"--generated-at",
default="2026-03-31",
help="Report date written to context_budget_summary.json.",
)
return parser.parse_args()
def main() -> None:
args = parse_args()
rows = []
for target in TARGETS:
report = analyze_skill(target["path"])
@@ -48,7 +60,7 @@ def main() -> None:
)
summary = {
"generated_at": "2026-03-31",
"generated_at": args.generated_at,
"targets": rows,
}
(ROOT / "reports").mkdir(parents=True, exist_ok=True)
scripts/render_review_annotations.py
+399
View File
@@ -0,0 +1,399 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
from datetime import date
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_SOURCE = ROOT / "reports" / "review_annotations_input.json"
DEFAULT_OUTPUT_JSON = ROOT / "reports" / "review_annotations.json"
DEFAULT_OUTPUT_MD = ROOT / "reports" / "review_annotations.md"
VALID_GATES = {
"intent-canvas",
"trigger-lab",
"output-lab",
"context-budget",
"runtime-matrix",
"trust-report",
"permission-gates",
"permission-runtime",
"skill-atlas",
"operations-loop",
"review-waivers",
"registry-audit",
"release-notes",
}
VALID_SEVERITIES = {"info", "note", "warning", "blocker"}
VALID_STATUSES = {"open", "resolved", "deferred"}
def display_path(root: Path, path: Path) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
try:
return str(path.resolve().relative_to(ROOT.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> tuple[dict[str, Any], list[str]]:
if not path.exists():
return {"schema_version": "1.0", "annotations": []}, []
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError as exc:
return {"schema_version": "1.0", "annotations": []}, [f"Invalid annotation JSON {display_path(ROOT, path)}: {exc}"]
if not isinstance(payload, dict):
return {"schema_version": "1.0", "annotations": []}, [f"Annotation file root must be an object: {display_path(ROOT, path)}"]
if not isinstance(payload.get("annotations", []), list):
return {"schema_version": "1.0", "annotations": []}, ["annotations must be a list"]
return payload, []
def template_payload() -> dict[str, Any]:
return {
"schema_version": "1.0",
"review_contract": {
"gate_key": "One Review Studio gate key, such as output-lab or trust-report.",
"target_path": "Relative path to the source or report being annotated.",
"line": "Optional positive integer line number.",
"severity": "info | note | warning | blocker.",
"status": "open | resolved | deferred.",
"body": "Reviewer-visible annotation text.",
},
"annotations": [],
}
def annotation_id(annotation: dict[str, Any]) -> str:
if annotation.get("id"):
return str(annotation["id"])
raw = "|".join(
[
str(annotation.get("gate_key", "")),
str(annotation.get("target_path", "")),
str(annotation.get("line", "")),
str(annotation.get("body", "")),
]
)
return "ann-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12]
def source_excerpt(path: Path, line: int | None) -> str:
if line is None or not path.exists() or not path.is_file():
return ""
try:
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
except OSError:
return ""
if line < 1 or line > len(lines):
return ""
return lines[line - 1].strip()[:220]
def normalize_annotation(skill_dir: Path, raw: dict[str, Any], index: int) -> tuple[dict[str, Any], list[str]]:
failures: list[str] = []
if not isinstance(raw, dict):
return {}, [f"annotation #{index} must be an object"]
gate_key = str(raw.get("gate_key", "")).strip()
if gate_key not in VALID_GATES:
failures.append(f"{annotation_id(raw)}: invalid gate_key {gate_key!r}")
severity = str(raw.get("severity", "note")).strip().lower()
if severity not in VALID_SEVERITIES:
failures.append(f"{annotation_id(raw)}: invalid severity {severity!r}")
status = str(raw.get("status", "open")).strip().lower()
if status not in VALID_STATUSES:
failures.append(f"{annotation_id(raw)}: invalid status {status!r}")
body = str(raw.get("body", "")).strip()
if not body:
failures.append(f"{annotation_id(raw)}: body is required")
target_path = str(raw.get("target_path", "")).strip()
if not target_path:
failures.append(f"{annotation_id(raw)}: target_path is required")
rel = Path(target_path)
if rel.is_absolute():
failures.append(f"{annotation_id(raw)}: target_path must be relative")
target = skill_dir / "__invalid_absolute_path__"
else:
target = (skill_dir / rel).resolve()
try:
target.relative_to(skill_dir.resolve())
except ValueError:
failures.append(f"{annotation_id(raw)}: target_path escapes skill directory")
target = skill_dir / "__invalid_escaped_path__"
line_value = raw.get("line", None)
line: int | None
if line_value is None or line_value == "":
line = None
else:
try:
line = int(line_value)
except (TypeError, ValueError):
line = None
failures.append(f"{annotation_id(raw)}: line must be a positive integer")
else:
if line < 1:
failures.append(f"{annotation_id(raw)}: line must be a positive integer")
normalized = {
"id": annotation_id(raw),
"gate_key": gate_key,
"target_path": target_path,
"line": line,
"severity": severity,
"status": status,
"reviewer": str(raw.get("reviewer", "")).strip(),
"created_at": str(raw.get("created_at", "")).strip(),
"body": body,
"suggested_action": str(raw.get("suggested_action", "")).strip(),
"evidence": str(raw.get("evidence", "")).strip(),
"target_exists": target.exists(),
"source_excerpt": source_excerpt(target, line),
}
return normalized, failures
def append_annotation(source: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]:
annotations = list(source.get("annotations", []))
raw = {
"id": args.annotation_id,
"gate_key": args.gate_key,
"target_path": args.target_path,
"line": args.line,
"severity": args.severity,
"status": args.status,
"reviewer": args.reviewer,
"created_at": args.created_at or str(date.today()),
"body": args.body,
"suggested_action": args.suggested_action,
"evidence": args.evidence,
}
annotations.append({key: value for key, value in raw.items() if value is not None and value != ""})
updated = dict(source)
updated["annotations"] = annotations
updated.setdefault("schema_version", "1.0")
return updated
def build_summary(annotations: list[dict[str, Any]], failures: list[str]) -> dict[str, Any]:
open_items = [item for item in annotations if item["status"] == "open"]
resolved_items = [item for item in annotations if item["status"] == "resolved"]
deferred_items = [item for item in annotations if item["status"] == "deferred"]
open_blockers = [item for item in open_items if item["severity"] == "blocker"]
open_warnings = [item for item in open_items if item["severity"] == "warning"]
return {
"annotation_count": len(annotations),
"open_count": len(open_items),
"resolved_count": len(resolved_items),
"deferred_count": len(deferred_items),
"open_blocker_count": len(open_blockers),
"open_warning_count": len(open_warnings),
"linked_gate_count": len({item["gate_key"] for item in annotations if item["gate_key"]}),
"target_missing_count": sum(1 for item in annotations if not item["target_exists"]),
"failure_count": len(failures),
}
def render_markdown(payload: dict[str, Any]) -> str:
summary = payload["summary"]
lines = [
"# Review Annotations",
"",
"This report renders reviewer annotations attached to Review Studio gates and source/report paths.",
"",
f"- Annotations: `{summary['annotation_count']}`",
f"- Open: `{summary['open_count']}`",
f"- Resolved: `{summary['resolved_count']}`",
f"- Deferred: `{summary['deferred_count']}`",
f"- Open blockers: `{summary['open_blocker_count']}`",
f"- Open warnings: `{summary['open_warning_count']}`",
"",
]
if not payload["annotations"]:
lines.extend(["No reviewer annotations recorded yet.", ""])
else:
lines.extend(
[
"## Annotation Ledger",
"",
"| ID | Gate | Severity | Status | Target | Reviewer | Note |",
"| --- | --- | --- | --- | --- | --- | --- |",
]
)
for item in payload["annotations"]:
line = f":{item['line']}" if item.get("line") else ""
note = item["body"].replace("|", "\\|")
lines.append(
f"| {item['id']} | {item['gate_key']} | {item['severity']} | {item['status']} | "
f"{item['target_path']}{line} | {item.get('reviewer', '')} | {note} |"
)
if item.get("source_excerpt"):
excerpt = str(item["source_excerpt"]).replace("|", "\\|")
lines.append(f"| | | | | | excerpt | `{excerpt}` |")
if payload.get("failures"):
lines.extend(["", "## Failures", ""])
for failure in payload["failures"]:
lines.append(f"- {failure}")
lines.extend(
[
"",
"## Review Rule",
"",
"- Use annotations for reviewer comments tied to a gate or source line.",
"- Use waivers only for explicit acceptance of warning-level release risk.",
"- Open blocker annotations should block a release decision until resolved or deferred with rationale.",
]
)
return "\n".join(lines).strip() + "\n"
def render_review_annotations(
skill_dir: Path,
source_json: Path,
output_json: Path,
output_md: Path,
args: argparse.Namespace,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
source, failures = load_json(source_json)
template_written = False
should_write_source = False
if args.write_template and not source_json.exists():
source_json.parent.mkdir(parents=True, exist_ok=True)
source_json.write_text(json.dumps(template_payload(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
source = template_payload()
template_written = True
if args.add_annotation:
source = append_annotation(source, args)
should_write_source = True
annotations: list[dict[str, Any]] = []
seen = set()
for index, raw in enumerate(source.get("annotations", []), start=1):
normalized, item_failures = normalize_annotation(skill_dir, raw, index)
failures.extend(item_failures)
if normalized:
if normalized["id"] in seen:
failures.append(f"{normalized['id']}: duplicate annotation id")
seen.add(normalized["id"])
annotations.append(normalized)
summary = build_summary(annotations, failures)
if should_write_source and not failures:
source_json.parent.mkdir(parents=True, exist_ok=True)
source_json.write_text(json.dumps(source, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
payload = {
"schema_version": "1.0",
"ok": not failures,
"skill_dir": display_path(ROOT, skill_dir),
"source": display_path(skill_dir, source_json),
"summary": summary,
"annotations": annotations,
"failures": failures,
"template_written": template_written,
"artifacts": {
"json": display_path(skill_dir, output_json),
"markdown": display_path(skill_dir, output_md),
},
}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(payload), encoding="utf-8")
return payload
def render_report(
skill_dir: Path,
annotations_json: Path | None = None,
output_json: Path | None = None,
output_md: Path | None = None,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
args = argparse.Namespace(
write_template=False,
add_annotation=False,
annotation_id=None,
gate_key=None,
target_path=None,
line=None,
severity="note",
status="open",
reviewer=None,
created_at=None,
body=None,
suggested_action=None,
evidence=None,
)
return render_review_annotations(
skill_dir,
annotations_json or skill_dir / "reports" / "review_annotations_input.json",
output_json or skill_dir / "reports" / "review_annotations.json",
output_md or skill_dir / "reports" / "review_annotations.md",
args,
)
def main() -> None:
parser = argparse.ArgumentParser(description="Render reviewer annotations for Review Studio gates and source paths.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--annotations-json")
parser.add_argument("--output-json")
parser.add_argument("--output-md")
parser.add_argument("--write-template", action="store_true")
parser.add_argument("--add-annotation", action="store_true")
parser.add_argument("--annotation-id")
parser.add_argument("--gate-key", choices=sorted(VALID_GATES))
parser.add_argument("--target-path")
parser.add_argument("--line", type=int)
parser.add_argument("--severity", choices=sorted(VALID_SEVERITIES), default="note")
parser.add_argument("--status", choices=sorted(VALID_STATUSES), default="open")
parser.add_argument("--reviewer")
parser.add_argument("--created-at")
parser.add_argument("--body")
parser.add_argument("--suggested-action")
parser.add_argument("--evidence")
args = parser.parse_args()
skill_dir = Path(args.skill_dir).resolve()
annotations_json = Path(args.annotations_json).resolve() if args.annotations_json else skill_dir / "reports" / "review_annotations_input.json"
output_json = Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "review_annotations.json"
output_md = Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "review_annotations.md"
if args.add_annotation:
missing = [name for name in ("gate_key", "target_path", "body") if not getattr(args, name)]
if missing:
payload = {
"schema_version": "1.0",
"ok": False,
"skill_dir": display_path(ROOT, skill_dir),
"source": display_path(skill_dir, annotations_json),
"summary": build_summary([], missing),
"annotations": [],
"failures": [f"--add-annotation requires {', '.join(missing)}"],
"template_written": False,
"artifacts": {
"json": display_path(skill_dir, output_json),
"markdown": display_path(skill_dir, output_md),
},
}
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(2)
payload = render_review_annotations(skill_dir, annotations_json, output_json, output_md, args)
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 2)
if __name__ == "__main__":
main()
scripts/render_review_studio.py
+824 -18
View File
File diff suppressed because it is too large Load Diff
scripts/render_review_waivers.py
+281
View File
@@ -0,0 +1,281 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
from datetime import date
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parent.parent
KNOWN_GATE_KEYS = {
"intent-canvas",
"trigger-lab",
"output-lab",
"context-budget",
"runtime-matrix",
"trust-report",
"permission-gates",
"permission-runtime",
"skill-atlas",
"operations-loop",
"registry-audit",
"release-notes",
}
VALID_DECISIONS = {"accepted-risk", "false-positive", "temporary-exception"}
MIN_REASON_CHARS = 20
def display_path(path: Path) -> str:
try:
return str(path.resolve().relative_to(ROOT.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def today_from(value: str | None) -> date:
if not value:
return date.today()
return date.fromisoformat(value[:10])
def parse_date(value: str) -> date | None:
try:
return date.fromisoformat(value[:10])
except (TypeError, ValueError):
return None
def waiver_id(entry: dict[str, Any]) -> str:
raw = "|".join(
[
str(entry.get("gate_key", "")),
str(entry.get("reviewer", "")),
str(entry.get("created_at", "")),
str(entry.get("reason", "")),
]
)
return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:12]
def normalize_waiver(entry: dict[str, Any]) -> dict[str, Any]:
normalized = {
"id": str(entry.get("id") or ""),
"gate_key": str(entry.get("gate_key") or ""),
"decision": str(entry.get("decision") or "accepted-risk"),
"reviewer": str(entry.get("reviewer") or ""),
"reason": str(entry.get("reason") or ""),
"created_at": str(entry.get("created_at") or ""),
"expires_at": str(entry.get("expires_at") or ""),
"evidence": str(entry.get("evidence") or ""),
"scope": str(entry.get("scope") or "current-release"),
}
normalized["id"] = normalized["id"] or waiver_id(normalized)
return normalized
def add_waiver(existing: list[dict[str, Any]], args: argparse.Namespace, today: date) -> list[dict[str, Any]]:
created_at = args.created_at or today.isoformat()
entry = normalize_waiver(
{
"gate_key": args.gate_key,
"decision": args.decision,
"reviewer": args.reviewer,
"reason": args.reason,
"created_at": created_at,
"expires_at": args.expires_at,
"evidence": args.evidence or "",
"scope": args.scope,
}
)
return [*existing, entry]
def validate_waivers(waivers: list[dict[str, Any]], today: date) -> tuple[list[dict[str, Any]], list[str], list[str]]:
normalized = []
failures = []
warnings = []
seen_ids = set()
for index, raw in enumerate(waivers, start=1):
entry = normalize_waiver(raw)
entry_failures = []
entry_warnings = []
if entry["id"] in seen_ids:
entry_failures.append("duplicate waiver id")
seen_ids.add(entry["id"])
if entry["gate_key"] not in KNOWN_GATE_KEYS:
entry_failures.append(f"unknown gate_key: {entry['gate_key'] or '<empty>'}")
if entry["decision"] not in VALID_DECISIONS:
entry_failures.append(f"invalid decision: {entry['decision'] or '<empty>'}")
if not entry["reviewer"]:
entry_failures.append("reviewer is required")
if len(entry["reason"]) < MIN_REASON_CHARS:
entry_failures.append(f"reason must be at least {MIN_REASON_CHARS} characters")
created_at = parse_date(entry["created_at"])
if created_at is None:
entry_failures.append("created_at must be ISO date")
expires_at = parse_date(entry["expires_at"])
if expires_at is None:
entry_failures.append("expires_at must be ISO date")
elif expires_at < today:
entry_warnings.append("waiver is expired")
entry["status"] = "invalid" if entry_failures else ("expired" if entry_warnings else "active")
entry["validation"] = {"failures": entry_failures, "warnings": entry_warnings}
normalized.append(entry)
failures.extend([f"waiver {index} ({entry['id']}): {item}" for item in entry_failures])
warnings.extend([f"waiver {index} ({entry['id']}): {item}" for item in entry_warnings])
return normalized, failures, warnings
def render_report(
skill_dir: Path,
waivers_json: Path | None = None,
output_json: Path | None = None,
output_md: Path | None = None,
generated_at: str | None = None,
add_args: argparse.Namespace | None = None,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
reports = skill_dir / "reports"
reports.mkdir(parents=True, exist_ok=True)
output_json = output_json or reports / "review_waivers.json"
output_md = output_md or reports / "review_waivers.md"
source_json = waivers_json or output_json
today = today_from(generated_at)
payload = load_json(source_json)
raw_waivers = payload.get("waivers", []) if isinstance(payload.get("waivers", []), list) else []
if add_args is not None:
raw_waivers = add_waiver(raw_waivers, add_args, today)
waivers, failures, warnings = validate_waivers(raw_waivers, today)
active = [item for item in waivers if item["status"] == "active"]
expired = [item for item in waivers if item["status"] == "expired"]
invalid = [item for item in waivers if item["status"] == "invalid"]
covered_gate_keys = sorted({item["gate_key"] for item in active})
report = {
"schema_version": "1.0",
"ok": not failures,
"skill_dir": display_path(skill_dir),
"generated_at": generated_at or today.isoformat(),
"summary": {
"waiver_count": len(waivers),
"active_count": len(active),
"expired_count": len(expired),
"invalid_count": len(invalid),
"covered_gate_count": len(covered_gate_keys),
"covered_gate_keys": covered_gate_keys,
},
"policy": {
"blocker_waivers_allowed": False,
"minimum_reason_chars": MIN_REASON_CHARS,
"expires_required": True,
"known_gate_keys": sorted(KNOWN_GATE_KEYS),
},
"waivers": waivers,
"failures": failures,
"warnings": warnings,
"artifacts": {
"json": display_path(output_json),
"markdown": display_path(output_md),
},
}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
return report
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
"# Review Waivers",
"",
f"- OK: `{report['ok']}`",
f"- Waivers: `{summary['waiver_count']}`",
f"- Active: `{summary['active_count']}`",
f"- Expired: `{summary['expired_count']}`",
f"- Invalid: `{summary['invalid_count']}`",
f"- Covered gates: `{', '.join(summary['covered_gate_keys']) or 'none'}`",
"",
"## Policy",
"",
"- Blocker waivers allowed: `False`",
f"- Minimum reason chars: `{report['policy']['minimum_reason_chars']}`",
"- Expiry is required for every waiver.",
"",
"## Waivers",
"",
]
if not report["waivers"]:
lines.append("- None")
else:
lines.extend(["| ID | Gate | Decision | Reviewer | Status | Expires | Reason |", "| --- | --- | --- | --- | --- | --- | --- |"])
for item in report["waivers"]:
reason = str(item["reason"]).replace("|", "\\|")
lines.append(
f"| `{item['id']}` | `{item['gate_key']}` | `{item['decision']}` | {item['reviewer']} | `{item['status']}` | `{item['expires_at']}` | {reason} |"
)
lines.extend(["", "## Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
lines.extend(["", "## Warnings", ""])
lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"])
return "\n".join(lines).strip() + "\n"
def main() -> None:
parser = argparse.ArgumentParser(description="Render or update Review Studio waiver evidence.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--waivers-json")
parser.add_argument("--output-json")
parser.add_argument("--output-md")
parser.add_argument("--generated-at")
parser.add_argument("--add-waiver", action="store_true")
parser.add_argument("--gate-key", choices=sorted(KNOWN_GATE_KEYS))
parser.add_argument("--decision", choices=sorted(VALID_DECISIONS), default="accepted-risk")
parser.add_argument("--reviewer")
parser.add_argument("--reason")
parser.add_argument("--expires-at")
parser.add_argument("--created-at")
parser.add_argument("--evidence")
parser.add_argument("--scope", default="current-release")
args = parser.parse_args()
add_args = None
if args.add_waiver:
required = {
"--gate-key": args.gate_key,
"--reviewer": args.reviewer,
"--reason": args.reason,
"--expires-at": args.expires_at,
}
missing = [name for name, value in required.items() if not value]
if missing:
print(json.dumps({"ok": False, "failures": [f"Missing required fields for --add-waiver: {', '.join(missing)}"]}, ensure_ascii=False, indent=2))
raise SystemExit(2)
add_args = args
payload = render_report(
Path(args.skill_dir),
waivers_json=Path(args.waivers_json).resolve() if args.waivers_json else None,
output_json=Path(args.output_json).resolve() if args.output_json else None,
output_md=Path(args.output_md).resolve() if args.output_md else None,
generated_at=args.generated_at,
add_args=add_args,
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 2)
if __name__ == "__main__":
main()
scripts/render_skill_overview.py
+54
View File
@@ -69,6 +69,14 @@ TEXT_EN = {
"这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。": "Use this report to quickly understand the generated skill's role, principles, trigger boundary, and deliverables.",
"先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。": "Clarify the recurring job, real input shape, and deliverable output before adding references, scripts, or evals.",
"如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。": "If the request is still fuzzy, tighten the boundary through intent dialogue before expanding the package.",
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。": "Output Review Adjudication is generated to record blind-review decisions, agreement rate, and pending cases.",
"已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。": "Output Execution Runs is generated to distinguish recorded fixtures, command runs, and model-run evidence.",
"尚未生成盲评审定报告。": "The blind review adjudication report has not been generated yet.",
"尚未生成输出执行证据报告。": "The output execution evidence report has not been generated yet.",
"先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。": "Record the reviewer's A/B choice before opening the answer key and calculating agreement.",
"缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。": "When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.",
"recorded fixture 只能证明可复现样本，不等同于模型执行。": "A recorded fixture proves reproducible samples only; it is not model execution.",
"只有 provider runner 返回 model metadata 时才计入 model-executed。": "Only provider runners that return model metadata count as model-executed.",
}
MODE_ZH = {
@@ -289,6 +297,10 @@ def render_html(summary: dict) -> str:
risk = summary.get("risk_governance", {})
assets = summary.get("package_assets", {})
roadmap = summary.get("iteration_roadmap", {})
output_execution = summary.get("output_execution", {})
output_execution_summary = output_execution.get("summary", {})
output_review = summary.get("output_review_adjudication", {})
output_review_summary = output_review.get("summary", {})
hero_meta = [
(f"技能名称：{summary['name']}", f"Skill name: {summary['name']}"),
(f"成熟度：{mode_zh(metadata.get('maturity_tier', 'scaffold'))}", f"Maturity: {metadata.get('maturity_tier', 'scaffold')}"),
@@ -312,6 +324,25 @@ def render_html(summary: dict) -> str:
[{"name": "强项", "signal": item, "response": "保留并复用"} for item in quality.get("strengths", [])[:3]]
+ [{"name": "缺口", "signal": item, "response": "纳入下一轮修复"} for item in quality.get("gaps", [])[:3]]
)
if output_review_summary:
agreement = output_review_summary.get("agreement_rate")
review_items = [
f"评审进度：{output_review_summary.get('judgment_count', 0)} / {output_review_summary.get('pair_count', 0)}",
f"待评审：{output_review_summary.get('pending_count', 0)}",
f"一致率：{agreement if agreement is not None else '暂无'}",
f"非法决策：{output_review_summary.get('invalid_decision_count', 0)}",
]
else:
review_items = ["尚未生成盲评审定报告。"]
if output_execution_summary:
execution_items = [
f"变体运行：{output_execution_summary.get('variant_run_count', 0)}",
f"模型执行：{output_execution_summary.get('model_executed_count', 0)}",
f"记录样本：{output_execution_summary.get('recorded_fixture_count', 0)}",
f"Token 估算：{output_execution_summary.get('token_estimated_count', 0)}",
]
else:
execution_items = ["尚未生成输出执行证据报告。"]
capability_items = [
f"能力类型：{profile.get('task_family', 'Skill workflow')}",
f"成熟度：{profile.get('maturity', 'scaffold')}",
@@ -665,6 +696,9 @@ def render_html(summary: dict) -> str:
gap: 16px;
align-items: stretch;
}}
.quality-panels {{
margin-top: 16px;
}}
.metrics-stack {{
display: grid;
gap: 18px;
@@ -1042,6 +1076,26 @@ def render_html(summary: dict) -> str:
<thead><tr><th>{bi_span("类型", "Type")}</th><th>{bi_span("证据", "Evidence")}</th><th>{bi_span("建议", "Action")}</th></tr></thead>
<tbody>{quality_rows}</tbody>
</table>
<div class="two-col quality-panels">
<article class="panel">
<h3>{bi_span("执行证据", "Execution Evidence")}</h3>
{render_list(execution_items)}
</article>
<article class="panel">
<h3>{bi_span("盲评审定", "Blind Adjudication")}</h3>
{render_list(review_items)}
</article>
</div>
<div class="two-col quality-panels">
<article class="panel">
<h3>{bi_span("评审原则", "Review Rule")}</h3>
{render_list(["先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。", "缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。"])}
</article>
<article class="panel">
<h3>{bi_span("运行原则", "Run Rule")}</h3>
{render_list(["recorded fixture 只能证明可复现样本，不等同于模型执行。", "只有 provider runner 返回 model metadata 时才计入 model-executed。"])}
</article>
</div>
</div>
</div>
</section>
scripts/render_social_preview.py
+17 -4
View File
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
from __future__ import annotations
import argparse
from pathlib import Path
from PIL import Image, ImageDraw, ImageFont
@@ -123,6 +124,7 @@ def draw_png() -> None:
def draw_svg() -> None:
OUT_DIR.mkdir(parents=True, exist_ok=True)
svg = f"""<svg xmlns="http://www.w3.org/2000/svg" width="{WIDTH}" height="{HEIGHT}" viewBox="0 0 {WIDTH} {HEIGHT}">
<rect width="{WIDTH}" height="{HEIGHT}" fill="{BACKGROUND}"/>
<text x="92" y="142" font-family="Georgia, serif" font-size="74" fill="{INK}">YAO</text>
@@ -161,10 +163,21 @@ def draw_svg() -> None:
def main() -> None:
draw_png()
draw_svg()
print(f"Wrote {PNG_PATH}")
print(f"Wrote {SVG_PATH}")
parser = argparse.ArgumentParser(description="Render the repository social preview image assets.")
parser.add_argument(
"--format",
choices=["all", "png", "svg"],
default="all",
help="Asset format to render.",
)
args = parser.parse_args()
if args.format in {"all", "png"}:
draw_png()
print(f"Wrote {PNG_PATH}")
if args.format in {"all", "svg"}:
draw_svg()
print(f"Wrote {SVG_PATH}")
if __name__ == "__main__":
scripts/run_conformance_suite.py
+1 -1
View File
@@ -173,7 +173,7 @@ def check_target(skill_dir: Path, target: str, evidence: dict[str, Any]) -> dict
if target in {"openai", "claude", "generic"}:
add_check(checks, failures, bool(degradation.get(target)), f"{target} degradation note exists", f"Missing {target} degradation note")
else:
warnings.append(f"{target} uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0.")
warnings.append(f"{target} uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0.")
add_check(checks, failures, ir.get("schema_version") == "2.0.0", "Skill IR schema_version is 2.0.0", "Missing or invalid Skill IR")
add_check(checks, failures, ir.get("name") == name, "Skill IR name matches frontmatter", "Skill IR name does not match frontmatter")
scripts/run_output_eval.py
+185 -2
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
from pathlib import Path
from typing import Any
@@ -7,6 +8,7 @@ from typing import Any
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_CASES = ROOT / "evals" / "output" / "cases.jsonl"
BLIND_SEED = "yao-output-eval-blind-v1"
def display_path(path: Path) -> str:
@@ -115,6 +117,98 @@ def grade_case(case: dict[str, Any]) -> dict[str, Any]:
}
def blind_variant_order(case_id: str) -> list[str]:
digest = hashlib.sha256(f"{BLIND_SEED}:{case_id}".encode("utf-8")).hexdigest()
return ["baseline", "with_skill"] if int(digest[:2], 16) % 2 == 0 else ["with_skill", "baseline"]
def output_for_role(case: dict[str, Any], role: str) -> str:
return str(case.get("baseline_output" if role == "baseline" else "with_skill_output", ""))
def expected_role(case: dict[str, Any]) -> str:
review = case.get("human_review", {}) if isinstance(case.get("human_review"), dict) else {}
winner = str(review.get("expected_winner", "with_skill"))
return winner if winner in {"baseline", "with_skill"} else "with_skill"
def build_blind_review_pack(cases: list[dict[str, Any]], results: list[dict[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]:
result_by_id = {item["id"]: item for item in results}
pairs = []
answer_pairs = []
for case in cases:
case_id = str(case["id"])
order = blind_variant_order(case_id)
variant_a_role, variant_b_role = order
expected = expected_role(case)
expected_variant = "A" if variant_a_role == expected else "B"
assertions = case.get("assertions", []) if isinstance(case.get("assertions"), list) else []
rubric = [
{
"id": str(item.get("id", "assertion")),
"description": str(item.get("description", "")),
"weight": float(item.get("weight", 1) or 0),
}
for item in assertions
if isinstance(item, dict)
]
pairs.append(
{
"case_id": case_id,
"prompt": str(case.get("prompt", "")),
"input_files": case.get("input_files", []),
"metadata": case.get("metadata", {}),
"review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.",
"rubric": rubric,
"variant_a": {
"blind_id": f"{case_id}:A",
"output": output_for_role(case, variant_a_role),
},
"variant_b": {
"blind_id": f"{case_id}:B",
"output": output_for_role(case, variant_b_role),
},
}
)
scored = result_by_id.get(case_id, {})
answer_pairs.append(
{
"case_id": case_id,
"variant_a_role": variant_a_role,
"variant_b_role": variant_b_role,
"expected_winner_role": expected,
"expected_winner_variant": expected_variant,
"score_winner_role": scored.get("winner", ""),
"delta": scored.get("delta", 0),
}
)
pack = {
"schema_version": "1.0",
"seed": BLIND_SEED,
"summary": {
"pair_count": len(pairs),
"answer_key_separate": True,
"with_skill_hidden_count": sum(
1
for pair in answer_pairs
if pair["variant_a_role"] == "with_skill" or pair["variant_b_role"] == "with_skill"
),
},
"pairs": pairs,
}
answer_key = {
"schema_version": "1.0",
"seed": BLIND_SEED,
"summary": {
"pair_count": len(answer_pairs),
"with_skill_expected_count": sum(1 for pair in answer_pairs if pair["expected_winner_role"] == "with_skill"),
"baseline_expected_count": sum(1 for pair in answer_pairs if pair["expected_winner_role"] == "baseline"),
},
"answers": answer_pairs,
}
return pack, answer_key
def build_summary(results: list[dict[str, Any]]) -> dict[str, Any]:
case_count = len(results)
baseline_average = sum(item["baseline"]["score"] for item in results) / case_count if case_count else 0
@@ -150,8 +244,12 @@ def render_markdown(payload: dict[str, Any]) -> str:
f"- With-skill pass rate: `{summary['with_skill_pass_rate']}`",
f"- Delta: `{summary['delta']}`",
f"- Regressions: `{summary['regression_count']}`",
f"- Blind A/B pairs: `{summary.get('blind_pair_count', 0)}`",
f"- Gate pass: `{summary['gate_pass']}`",
"",
"Blind review artifacts are generated separately so reviewers can inspect A/B outputs without seeing the answer key.",
"Run output review adjudication after reviewer decisions are recorded; pending cases should stay pending rather than being counted as human agreement.",
"",
"## Case Results",
"",
"| Case | Baseline | With Skill | Delta | Winner | Failed With-Skill Assertions |",
@@ -181,10 +279,69 @@ def render_markdown(payload: dict[str, Any]) -> str:
return "\n".join(lines).strip() + "\n"
def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dict[str, Any]:
def render_blind_review_markdown(pack: dict[str, Any]) -> str:
summary = pack["summary"]
lines = [
"# Output Blind A/B Review Pack",
"",
"This packet hides whether each variant came from the baseline or the skill-guided output. Use the separate answer key only after review.",
"",
f"- Pairs: `{summary['pair_count']}`",
f"- Seed: `{pack['seed']}`",
f"- Answer key separate: `{summary['answer_key_separate']}`",
"",
]
for pair in pack["pairs"]:
lines.extend(
[
f"## Case: {pair['case_id']}",
"",
f"Prompt: {pair['prompt']}",
"",
"Rubric:",
]
)
for item in pair["rubric"]:
lines.append(f"- `{item['id']}` ({item['weight']}): {item['description']}")
lines.extend(
[
"",
"### Variant A",
"",
str(pair["variant_a"]["output"]),
"",
"### Variant B",
"",
str(pair["variant_b"]["output"]),
"",
]
)
return "\n".join(lines).strip() + "\n"
def run_output_eval(
cases_path: Path,
output_json: Path,
output_md: Path,
blind_pack_json: Path,
blind_pack_md: Path,
blind_answer_key_json: Path,
) -> dict[str, Any]:
cases = load_cases(cases_path)
validation_failures = [failure for case in cases for failure in validate_case(case, cases_path.parent)]
if validation_failures:
blind_pack = {
"schema_version": "1.0",
"seed": BLIND_SEED,
"summary": {"pair_count": 0, "answer_key_separate": True, "with_skill_hidden_count": 0},
"pairs": [],
}
blind_answer_key = {
"schema_version": "1.0",
"seed": BLIND_SEED,
"summary": {"pair_count": 0, "with_skill_expected_count": 0, "baseline_expected_count": 0},
"answers": [],
}
payload = {
"ok": False,
"cases": display_path(cases_path),
@@ -195,6 +352,7 @@ def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dic
"delta": 0,
"regression_count": 0,
"gate_pass": False,
"blind_pair_count": 0,
"failure_taxonomy": ["invalid_case"],
},
"results": [],
@@ -202,6 +360,7 @@ def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dic
}
else:
results = [grade_case(case) for case in cases]
blind_pack, blind_answer_key = build_blind_review_pack(cases, results)
payload = {
"ok": True,
"cases": display_path(cases_path),
@@ -209,11 +368,29 @@ def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dic
"results": results,
"failures": [],
}
payload["artifacts"] = {"json": display_path(output_json), "markdown": display_path(output_md)}
payload["summary"]["blind_pair_count"] = blind_pack["summary"]["pair_count"]
payload["blind_review"] = {
"pack": display_path(blind_pack_json),
"answer_key": display_path(blind_answer_key_json),
"pair_count": blind_pack["summary"]["pair_count"],
}
payload["artifacts"] = {
"json": display_path(output_json),
"markdown": display_path(output_md),
"blind_review_pack_json": display_path(blind_pack_json),
"blind_review_pack_md": display_path(blind_pack_md),
"blind_answer_key_json": display_path(blind_answer_key_json),
}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
blind_pack_json.parent.mkdir(parents=True, exist_ok=True)
blind_pack_md.parent.mkdir(parents=True, exist_ok=True)
blind_answer_key_json.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(payload), encoding="utf-8")
blind_pack_json.write_text(json.dumps(blind_pack, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
blind_pack_md.write_text(render_blind_review_markdown(blind_pack), encoding="utf-8")
blind_answer_key_json.write_text(json.dumps(blind_answer_key, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
return payload
@@ -222,12 +399,18 @@ def main() -> None:
parser.add_argument("--cases", default=str(DEFAULT_CASES))
parser.add_argument("--output-json", default=str(ROOT / "reports" / "output_quality_scorecard.json"))
parser.add_argument("--output-md", default=str(ROOT / "reports" / "output_quality_scorecard.md"))
parser.add_argument("--blind-pack-json", default=str(ROOT / "reports" / "output_blind_review_pack.json"))
parser.add_argument("--blind-pack-md", default=str(ROOT / "reports" / "output_blind_review_pack.md"))
parser.add_argument("--blind-answer-key-json", default=str(ROOT / "reports" / "output_blind_answer_key.json"))
args = parser.parse_args()
payload = run_output_eval(
Path(args.cases).resolve(),
Path(args.output_json).resolve(),
Path(args.output_md).resolve(),
Path(args.blind_pack_json).resolve(),
Path(args.blind_pack_md).resolve(),
Path(args.blind_answer_key_json).resolve(),
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 2)
scripts/run_output_execution.py
+382
View File
@@ -0,0 +1,382 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import shlex
import subprocess
import time
from pathlib import Path
from typing import Any
from run_output_eval import DEFAULT_CASES, display_path, grade_output, load_cases, validate_case
ROOT = Path(__file__).resolve().parent.parent
DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_execution_runs.json"
DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_execution_runs.md"
VARIANTS = ("baseline", "with_skill")
def output_for_variant(case: dict[str, Any], variant: str) -> str:
return str(case.get("baseline_output" if variant == "baseline" else "with_skill_output", ""))
def output_key(variant: str) -> str:
return "baseline_output" if variant == "baseline" else "with_skill_output"
def parse_runner_command(value: str | None) -> list[str]:
if not value:
return []
stripped = value.strip()
if stripped.startswith("["):
payload = json.loads(stripped)
if not isinstance(payload, list) or not all(isinstance(item, str) and item for item in payload):
raise ValueError("--runner-command JSON must be a non-empty string list")
return payload
return shlex.split(stripped)
def display_command(command: list[str]) -> list[str]:
displayed: list[str] = []
for item in command:
path = Path(item)
if path.is_absolute() and path.exists():
displayed.append(display_path(path))
else:
displayed.append(item)
return displayed
def estimate_tokens(text: str) -> int:
if not text:
return 0
return max(1, round(len(text) / 4))
def parse_runner_stdout(stdout: str) -> tuple[str, dict[str, Any]]:
stripped = stdout.strip()
if not stripped:
return "", {}
try:
payload = json.loads(stripped)
except json.JSONDecodeError:
return stdout, {}
if not isinstance(payload, dict):
return stdout, {}
return str(payload.get("output", "")), payload
def usage_payload(payload: dict[str, Any], prompt: str, output: str) -> dict[str, Any]:
usage = payload.get("usage", {}) if isinstance(payload.get("usage"), dict) else {}
input_tokens = usage.get("input_tokens", usage.get("prompt_tokens"))
output_tokens = usage.get("output_tokens", usage.get("completion_tokens"))
total_tokens = usage.get("total_tokens")
estimated = bool(usage.get("estimated", False))
if input_tokens is None:
input_tokens = estimate_tokens(prompt)
estimated = True
if output_tokens is None:
output_tokens = estimate_tokens(output)
estimated = True
if total_tokens is None:
total_tokens = int(input_tokens or 0) + int(output_tokens or 0)
estimated = True
return {
"input_tokens": int(input_tokens or 0),
"output_tokens": int(output_tokens or 0),
"total_tokens": int(total_tokens or 0),
"estimated": estimated,
}
def recorded_fixture_run(case: dict[str, Any], variant: str, assertions: list[dict[str, Any]]) -> dict[str, Any]:
output = output_for_variant(case, variant)
grade = grade_output(output, assertions)
return {
"case_id": str(case.get("id", "")),
"variant": variant,
"status": "pass",
"execution_mode": "recorded_fixture",
"model_executed": False,
"command_executed": False,
"duration_ms": None,
"provider": "",
"model": "",
"usage": usage_payload({}, str(case.get("prompt", "")), output),
"score": grade["score"],
"passed_count": grade["passed_count"],
"failed_count": grade["failed_count"],
"failed_assertions": [item["id"] for item in grade["failed"]],
"output_sha256": hashlib.sha256(output.encode("utf-8")).hexdigest(),
"failure": "",
}
def command_run(
case: dict[str, Any],
variant: str,
assertions: list[dict[str, Any]],
command: list[str],
timeout_seconds: float,
) -> dict[str, Any]:
request = {
"case_id": str(case.get("id", "")),
"variant": variant,
"prompt": str(case.get("prompt", "")),
"input_files": case.get("input_files", []),
"metadata": case.get("metadata", {}),
"fixture_output": output_for_variant(case, variant),
"output_key": output_key(variant),
}
started = time.perf_counter()
try:
proc = subprocess.run(
command,
input=json.dumps(request, ensure_ascii=False),
capture_output=True,
text=True,
timeout=timeout_seconds,
check=False,
)
duration_ms = round((time.perf_counter() - started) * 1000, 2)
except subprocess.TimeoutExpired as exc:
return {
"case_id": request["case_id"],
"variant": variant,
"status": "fail",
"execution_mode": "command",
"model_executed": False,
"command_executed": True,
"duration_ms": round(timeout_seconds * 1000, 2),
"provider": "",
"model": "",
"usage": usage_payload({}, request["prompt"], ""),
"score": 0,
"passed_count": 0,
"failed_count": len(assertions),
"failed_assertions": [str(item.get("id", "assertion")) for item in assertions],
"output_sha256": "",
"failure": f"runner timed out after {timeout_seconds}s: {exc}",
}
output, payload = parse_runner_stdout(proc.stdout)
grade = grade_output(output, assertions)
execution_kind = str(payload.get("execution_kind", "command"))
provider = str(payload.get("provider", ""))
model = str(payload.get("model", ""))
model_executed = execution_kind == "model" or bool(model and provider)
return {
"case_id": request["case_id"],
"variant": variant,
"status": "pass" if proc.returncode == 0 and output else "fail",
"execution_mode": execution_kind if execution_kind in {"command", "model"} else "command",
"model_executed": model_executed,
"command_executed": True,
"duration_ms": duration_ms,
"provider": provider,
"model": model,
"usage": usage_payload(payload, request["prompt"], output),
"score": grade["score"] if output else 0,
"passed_count": grade["passed_count"] if output else 0,
"failed_count": grade["failed_count"] if output else len(assertions),
"failed_assertions": [item["id"] for item in grade["failed"]] if output else [str(item.get("id", "assertion")) for item in assertions],
"output_sha256": hashlib.sha256(output.encode("utf-8")).hexdigest() if output else "",
"failure": "" if proc.returncode == 0 and output else (proc.stderr.strip() or "runner returned no output"),
}
def build_summary(runs: list[dict[str, Any]]) -> dict[str, Any]:
variant_run_count = len(runs)
case_ids = sorted({item["case_id"] for item in runs})
baseline = [item for item in runs if item["variant"] == "baseline"]
with_skill = [item for item in runs if item["variant"] == "with_skill"]
baseline_average = sum(item["score"] for item in baseline) / len(baseline) if baseline else 0
with_skill_average = sum(item["score"] for item in with_skill) / len(with_skill) if with_skill else 0
regression_count = sum(
1
for case_id in case_ids
if max((item["score"] for item in with_skill if item["case_id"] == case_id), default=0)
< max((item["score"] for item in baseline if item["case_id"] == case_id), default=0)
)
failure_count = sum(1 for item in runs if item["status"] != "pass")
command_executed_count = sum(1 for item in runs if item.get("command_executed"))
model_executed_count = sum(1 for item in runs if item.get("model_executed"))
recorded_fixture_count = sum(1 for item in runs if item.get("execution_mode") == "recorded_fixture")
timing_observed_count = sum(1 for item in runs if item.get("duration_ms") is not None)
token_estimated_count = sum(1 for item in runs if item.get("usage", {}).get("estimated"))
token_observed_count = variant_run_count - token_estimated_count
return {
"case_count": len(case_ids),
"variant_run_count": variant_run_count,
"command_executed_count": command_executed_count,
"model_executed_count": model_executed_count,
"recorded_fixture_count": recorded_fixture_count,
"timing_observed_count": timing_observed_count,
"token_observed_count": token_observed_count,
"token_estimated_count": token_estimated_count,
"baseline_pass_rate": round(baseline_average, 2),
"with_skill_pass_rate": round(with_skill_average, 2),
"delta": round(with_skill_average - baseline_average, 2),
"regression_count": regression_count,
"failure_count": failure_count,
"gate_pass": failure_count == 0 and with_skill_average >= baseline_average and regression_count == 0,
}
def render_markdown(payload: dict[str, Any]) -> str:
summary = payload["summary"]
lines = [
"# Output Execution Runs",
"",
"This report records how output-eval variants were produced and whether timing or token evidence is observed or estimated.",
"",
f"- Cases: `{summary['case_count']}`",
f"- Variant runs: `{summary['variant_run_count']}`",
f"- Command executed: `{summary['command_executed_count']}`",
f"- Model executed: `{summary['model_executed_count']}`",
f"- Recorded fixtures: `{summary['recorded_fixture_count']}`",
f"- Timing observed: `{summary['timing_observed_count']}`",
f"- Token observed: `{summary['token_observed_count']}`",
f"- Token estimated: `{summary['token_estimated_count']}`",
f"- Delta: `{summary['delta']}`",
f"- Gate pass: `{summary['gate_pass']}`",
"",
]
if summary["model_executed_count"] == 0:
lines.extend(
[
"No model-executed runs are recorded yet.",
"",
"Use `--runner-command` with a provider-backed runner to replace recorded fixtures with real model output evidence.",
"",
]
)
if summary["command_executed_count"] > 0:
lines.extend(
[
"Command runner evidence is present. This proves the eval harness executed an external command, but it is not provider-backed model evidence unless the runner reports model metadata.",
"",
]
)
lines.extend(
[
"## Runs",
"",
"| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |",
"| --- | --- | --- | --- | ---: | ---: | ---: | --- |",
]
)
for item in payload["runs"]:
usage = item.get("usage", {})
duration = "" if item.get("duration_ms") is None else str(item["duration_ms"])
model = item.get("model") or item.get("provider") or ""
lines.append(
f"| {item['case_id']} | {item['variant']} | {item['execution_mode']} | {model} | "
f"{duration} | {usage.get('total_tokens', 0)} | {item['score']} | {item['status']} |"
)
failures = [item for item in payload["runs"] if item.get("failure")]
if failures:
lines.extend(["", "## Failures", ""])
for item in failures:
lines.append(f"- `{item['case_id']}` `{item['variant']}`: {item['failure']}")
lines.extend(
[
"",
"## Next Fixes",
"",
"- Keep recorded fixtures as reproducible baselines, but do not describe them as model-executed evidence.",
"- Add provider-backed command runners for holdout cases when release confidence depends on real generation behavior.",
"- Compare timing, token cost, and assertion deltas before promoting a skill to governed reuse.",
]
)
return "\n".join(lines).strip() + "\n"
def run_output_execution(
cases_path: Path,
output_json: Path,
output_md: Path,
runner_command: list[str],
timeout_seconds: float,
) -> dict[str, Any]:
cases = load_cases(cases_path)
validation_failures = [failure for case in cases for failure in validate_case(case, cases_path.parent)]
runs: list[dict[str, Any]] = []
if not validation_failures:
for case in cases:
assertions = case.get("assertions", []) if isinstance(case.get("assertions"), list) else []
for variant in VARIANTS:
if runner_command:
runs.append(command_run(case, variant, assertions, runner_command, timeout_seconds))
else:
runs.append(recorded_fixture_run(case, variant, assertions))
summary = build_summary(runs)
failures = validation_failures + [f"{item['case_id']} {item['variant']}: {item['failure']}" for item in runs if item.get("failure")]
payload = {
"schema_version": "1.0",
"ok": not failures and summary["gate_pass"],
"cases": display_path(cases_path),
"runner": {
"mode": "command" if runner_command else "recorded_fixture",
"command": display_command(runner_command),
"timeout_seconds": timeout_seconds if runner_command else None,
},
"summary": summary,
"runs": runs,
"failures": failures,
"artifacts": {
"json": display_path(output_json),
"markdown": display_path(output_md),
},
}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(payload), encoding="utf-8")
return payload
def main() -> None:
parser = argparse.ArgumentParser(description="Record output-eval execution evidence for static, command, or model-backed runs.")
parser.add_argument("--cases", default=str(DEFAULT_CASES))
parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON))
parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD))
parser.add_argument("--runner-command", help="Command string or JSON string list. Receives a JSON request on stdin.")
parser.add_argument("--timeout-seconds", type=float, default=30.0)
args = parser.parse_args()
try:
runner_command = parse_runner_command(args.runner_command)
except (json.JSONDecodeError, ValueError) as exc:
payload = {
"schema_version": "1.0",
"ok": False,
"cases": display_path(Path(args.cases).resolve()),
"runner": {"mode": "invalid", "command": [], "timeout_seconds": args.timeout_seconds},
"summary": build_summary([]),
"runs": [],
"failures": [str(exc)],
"artifacts": {"json": display_path(Path(args.output_json).resolve()), "markdown": display_path(Path(args.output_md).resolve())},
}
output_json = Path(args.output_json).resolve()
output_md = Path(args.output_md).resolve()
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(payload), encoding="utf-8")
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(2)
payload = run_output_execution(
Path(args.cases).resolve(),
Path(args.output_json).resolve(),
Path(args.output_md).resolve(),
runner_command,
args.timeout_seconds,
)
print(json.dumps(payload, ensure_ascii=False, indent=2))
raise SystemExit(0 if payload["ok"] else 2)
if __name__ == "__main__":
main()
scripts/simulate_install.py
+271
View File
@@ -0,0 +1,271 @@
#!/usr/bin/env python3
import argparse
import json
import shutil
import tempfile
import zipfile
from datetime import date
from pathlib import Path, PurePosixPath
from typing import Any
try:
import yaml
except ImportError: # pragma: no cover
yaml = None
ROOT = Path(__file__).resolve().parent.parent
def display_path(path: Path, root: Path = ROOT) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def load_yaml(path: Path) -> dict[str, Any]:
if not path.exists() or yaml is None:
return {}
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
return payload if isinstance(payload, dict) else {}
def read_frontmatter(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
text = path.read_text(encoding="utf-8", errors="replace")
if not text.startswith("---"):
return {}
parts = text.split("---", 2)
if len(parts) < 3:
return {}
if yaml is not None:
payload = yaml.safe_load(parts[1]) or {}
return payload if isinstance(payload, dict) else {}
data: dict[str, Any] = {}
for line in parts[1].splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
data[key.strip()] = value.strip().strip('"')
return data
def zip_names(zip_path: Path) -> list[str]:
with zipfile.ZipFile(zip_path) as archive:
return archive.namelist()
def unsafe_zip_entries(names: list[str]) -> list[str]:
unsafe = []
for name in names:
pure = PurePosixPath(name)
if pure.is_absolute() or ".." in pure.parts or name.startswith("\\") or (pure.parts and ":" in pure.parts[0]):
unsafe.append(name)
return unsafe
def top_level_dirs(names: list[str]) -> list[str]:
roots = set()
for name in names:
parts = PurePosixPath(name).parts
if parts:
roots.add(parts[0])
return sorted(roots)
def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str, passed: bool, detail: str) -> None:
checks.append({"id": check_id, "status": "pass" if passed else "fail", "detail": detail})
if not passed:
failures.append(detail)
def adapter_targets(package_dir: Path) -> list[str]:
targets_dir = package_dir / "targets"
if not targets_dir.exists():
return []
return sorted(path.name for path in targets_dir.iterdir() if path.is_dir())
def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | None, generated_at: str) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
package_dir = package_dir.resolve()
checks: list[dict[str, str]] = []
failures: list[str] = []
warnings: list[str] = []
archive_path = package_dir / f"{skill_dir.name}.zip"
package_manifest = load_json(package_dir / "manifest.json")
installed_dir = Path("")
archive_entries: list[str] = []
temp_dir: tempfile.TemporaryDirectory[str] | None = None
if install_root is None:
temp_dir = tempfile.TemporaryDirectory(prefix="yao-skill-install-")
install_base = Path(temp_dir.name)
install_root_is_temp = True
else:
requested_root = install_root.resolve()
requested_root.mkdir(parents=True, exist_ok=True)
install_base = requested_root / f"simulate-{skill_dir.name}"
if install_base.exists():
shutil.rmtree(install_base)
install_base.mkdir(parents=True, exist_ok=True)
install_root_is_temp = False
try:
add_check(checks, failures, "archive-present", archive_path.exists(), f"Package archive exists: {display_path(archive_path)}")
if archive_path.exists():
try:
archive_entries = zip_names(archive_path)
except zipfile.BadZipFile:
add_check(checks, failures, "archive-readable", False, f"Archive is not a readable zip: {display_path(archive_path)}")
else:
unsafe_entries = unsafe_zip_entries(archive_entries)
add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries")
roots = top_level_dirs(archive_entries)
add_check(checks, failures, "single-top-level", roots == [skill_dir.name], f"Archive top-level directory is {skill_dir.name}")
if not unsafe_entries and roots:
with zipfile.ZipFile(archive_path) as archive:
archive.extractall(install_base)
installed_dir = install_base / roots[0]
frontmatter = read_frontmatter(installed_dir / "SKILL.md") if installed_dir else {}
source_manifest = load_json(installed_dir / "manifest.json") if installed_dir else {}
interface_doc = load_yaml(installed_dir / "agents" / "interface.yaml") if installed_dir else {}
add_check(checks, failures, "entrypoint-load", bool(frontmatter), "Installed SKILL.md frontmatter is readable")
add_check(checks, failures, "entrypoint-name", frontmatter.get("name") == skill_dir.name, "Installed SKILL.md name matches package directory")
add_check(checks, failures, "entrypoint-description", bool(frontmatter.get("description")), "Installed SKILL.md description is present")
add_check(checks, failures, "manifest-load", bool(source_manifest), "Installed manifest.json is readable")
add_check(checks, failures, "manifest-name", source_manifest.get("name") == package_manifest.get("name"), "Installed manifest name matches package manifest")
add_check(checks, failures, "manifest-version", source_manifest.get("version") == package_manifest.get("version"), "Installed manifest version matches package manifest")
add_check(checks, failures, "interface-load", bool(interface_doc.get("interface")), "Installed agents/interface.yaml is readable")
add_check(checks, failures, "overview-report", (installed_dir / "reports" / "skill-overview.html").exists(), "Installed overview report is present")
add_check(checks, failures, "review-studio-report", (installed_dir / "reports" / "review-studio.html").exists(), "Installed Review Studio report is present")
adapters = adapter_targets(package_dir)
for target in adapters:
adapter = load_json(package_dir / "targets" / target / "adapter.json")
add_check(checks, failures, f"adapter-{target}", bool(adapter), f"{target} adapter is readable after package install simulation")
add_check(
checks,
failures,
f"adapter-{target}-name",
adapter.get("name") == package_manifest.get("name"),
f"{target} adapter name matches package manifest",
)
if not adapters:
warnings.append("No target adapters found in package directory.")
if install_root_is_temp:
install_root_display = "[temporary-install-root]"
installed_dir_display = f"{install_root_display}/{installed_dir.name}" if installed_dir else ""
else:
install_root_display = display_path(install_base)
installed_dir_display = display_path(installed_dir) if installed_dir else ""
report = {
"ok": not failures,
"schema_version": "2.0",
"generated_at": generated_at,
"skill_dir": display_path(skill_dir),
"package_dir": display_path(package_dir),
"install_root": install_root_display,
"installed_skill_dir": installed_dir_display,
"summary": {
"archive_present": archive_path.exists(),
"archive_entry_count": len(archive_entries),
"archive_extracted": bool(installed_dir and installed_dir.exists()),
"entrypoint_loaded": bool(frontmatter),
"manifest_loaded": bool(source_manifest),
"interface_loaded": bool(interface_doc.get("interface")),
"adapter_count": len(adapters),
"install_root_is_temp": install_root_is_temp,
"failure_count": len(failures),
"warning_count": len(warnings),
},
"checks": checks,
"failures": failures,
"warnings": warnings,
"artifacts": {
"archive": display_path(archive_path) if archive_path.exists() else "",
"package_manifest": display_path(package_dir / "manifest.json"),
},
}
return report
finally:
if temp_dir is not None:
temp_dir.cleanup()
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
"# Install Simulation",
"",
f"- OK: `{report['ok']}`",
f"- Package directory: `{report['package_dir']}`",
f"- Archive extracted: `{summary['archive_extracted']}`",
f"- Entrypoint loaded: `{summary['entrypoint_loaded']}`",
f"- Manifest loaded: `{summary['manifest_loaded']}`",
f"- Interface loaded: `{summary['interface_loaded']}`",
f"- Adapters readable: `{summary['adapter_count']}`",
f"- Failures: `{summary['failure_count']}`",
f"- Warnings: `{summary['warning_count']}`",
"",
"## Checks",
"",
"| Check | Status | Detail |",
"| --- | --- | --- |",
]
for item in report["checks"]:
lines.append(f"| `{item['id']}` | `{item['status']}` | {item['detail']} |")
lines.extend(["", "## Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
lines.extend(["", "## Warnings", ""])
lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"])
return "\n".join(lines) + "\n"
def main() -> None:
parser = argparse.ArgumentParser(description="Simulate installing a generated skill package into a temporary local skill root.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--package-dir", default="dist")
parser.add_argument("--install-root")
parser.add_argument("--output-json", default="reports/install_simulation.json")
parser.add_argument("--output-md", default="reports/install_simulation.md")
parser.add_argument("--generated-at", default=str(date.today()))
args = parser.parse_args()
package_dir = Path(args.package_dir)
if not package_dir.is_absolute():
package_dir = Path.cwd() / package_dir
install_root = Path(args.install_root) if args.install_root else None
report = simulate_install(Path(args.skill_dir), package_dir, install_root, args.generated_at)
output_json = Path(args.output_json)
output_md = Path(args.output_md)
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
report["artifacts"]["json"] = display_path(output_json)
report["artifacts"]["markdown"] = display_path(output_md)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
raise SystemExit(0 if report["ok"] else 2)
if __name__ == "__main__":
main()
scripts/skill_report_charts.py
+3
View File
@@ -3,6 +3,9 @@ import html
import math
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to render inline SVG report charts."
BRAND = "#1B365D"
BORDER = "#e8e6dc"
SOFT = "#faf9f5"
scripts/skill_report_metrics.py
+10
View File
@@ -2,8 +2,12 @@
from pathlib import Path
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to calculate overview report metrics."
REPORT_EVIDENCE = [
"skill-ir.json",
"compiled_targets.json",
"intent-dialogue.json",
"intent-confidence.json",
"reference-synthesis.json",
@@ -11,6 +15,12 @@ REPORT_EVIDENCE = [
"conformance_matrix.json",
"security_trust_report.json",
"skill_atlas.json",
"registry_audit.json",
"package_verification.json",
"install_simulation.json",
"upgrade_check.json",
"adoption_drift_report.json",
"review_waivers.json",
"artifact-design-profile.json",
"prompt-quality-profile.json",
"system-model.json",
scripts/skill_report_model.py
+172 -11
View File
@@ -12,6 +12,9 @@ except ImportError: # pragma: no cover
yaml = None
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to build the v2 report data model."
KNOWN_ENTRIES = [
("SKILL.md", "Skill entrypoint"),
("README.md", "Human-readable usage guide"),
@@ -194,14 +197,30 @@ def derive_strengths(skill_dir: Path, metadata: dict) -> list[str]:
strengths = ["触发面保持精简，并锚定在 frontmatter description。"]
if (skill_dir / "reports" / "skill-ir.json").exists() or (skill_dir / "skill-ir" / "examples").exists():
strengths.append("已生成 Skill IR，核心语义可先于平台打包被审查和迁移。")
if (skill_dir / "reports" / "compiled_targets.json").exists():
strengths.append("已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。")
if (skill_dir / "reports" / "output_quality_scorecard.json").exists():
strengths.append("已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。")
if (skill_dir / "reports" / "output_execution_runs.json").exists():
strengths.append("已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。")
if (skill_dir / "reports" / "output_review_adjudication.json").exists():
strengths.append("已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。")
if (skill_dir / "reports" / "conformance_matrix.json").exists():
strengths.append("已生成 Runtime Conformance Matrix，可审查目标端消费能力。")
if (skill_dir / "reports" / "security_trust_report.json").exists():
strengths.append("已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。")
if (skill_dir / "reports" / "skill_atlas.json").exists():
strengths.append("已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。")
if (skill_dir / "reports" / "registry_audit.json").exists():
strengths.append("已生成 Registry Audit，可审查版本、owner、license、checksum 和目标兼容矩阵。")
if (skill_dir / "reports" / "install_simulation.json").exists():
strengths.append("已生成 Install Simulation，可审查 zip 解压、入口加载、接口元数据和 adapter 可读性。")
if (skill_dir / "reports" / "adoption_drift_report.json").exists():
strengths.append("已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。")
if (skill_dir / "reports" / "review_waivers.json").exists():
strengths.append("已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。")
if (skill_dir / "reports" / "review_annotations.json").exists():
strengths.append("已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。")
if (skill_dir / "reports" / "review-studio.json").exists():
strengths.append("已生成 Review Studio 2.0，可在一页中查看 blocker、warning、证据路径和发布闸门。")
if (skill_dir / "agents" / "interface.yaml").exists():
@@ -454,9 +473,21 @@ def build_report_model(skill_dir: Path) -> dict:
system_model = load_json(skill_dir / "reports" / "system-model.json")
output_risk = load_json(skill_dir / "reports" / "output-risk-profile.json")
output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json")
output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json")
output_blind_review = load_json(skill_dir / "reports" / "output_blind_review_pack.json")
output_review_adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json")
conformance = load_json(skill_dir / "reports" / "conformance_matrix.json")
runtime_permissions = load_json(skill_dir / "reports" / "runtime_permission_probes.json")
trust_report = load_json(skill_dir / "reports" / "security_trust_report.json")
skill_atlas = load_json(skill_dir / "reports" / "skill_atlas.json")
registry_audit = load_json(skill_dir / "reports" / "registry_audit.json")
package_verification = load_json(skill_dir / "reports" / "package_verification.json")
install_simulation = load_json(skill_dir / "reports" / "install_simulation.json")
upgrade_check = load_json(skill_dir / "reports" / "upgrade_check.json")
adoption_drift = load_json(skill_dir / "reports" / "adoption_drift_report.json")
review_waivers = load_json(skill_dir / "reports" / "review_waivers.json")
review_annotations = load_json(skill_dir / "reports" / "review_annotations.json")
compiled_targets = load_json(skill_dir / "reports" / "compiled_targets.json")
skill_ir = load_json(skill_dir / "reports" / "skill-ir.json")
if not skill_ir:
example_ir = skill_dir / "skill-ir" / "examples" / f"{frontmatter.get('name', skill_dir.name)}.json"
@@ -485,6 +516,41 @@ def build_report_model(skill_dir: Path) -> dict:
"skill_archetype": manifest.get("skill_archetype", manifest.get("maturity_tier", "scaffold")),
"updated_at": manifest.get("updated_at", str(date.today())),
}
deliverables = [
"SKILL.md",
"agents/interface.yaml",
"reports/skill-ir.json",
"reports/compiled_targets.md",
"reports/output_quality_scorecard.md",
"reports/conformance_matrix.md",
"reports/security_trust_report.md",
"reports/skill_atlas.html",
"reports/registry_audit.md",
"reports/package_verification.md",
"reports/install_simulation.md",
"reports/upgrade_check.md",
"reports/adoption_drift_report.md",
"reports/review_waivers.md",
"reports/review_annotations.md",
"reports/review-studio.html",
"reports/skill-overview.html",
]
if (skill_dir / "reports" / "runtime_permission_probes.md").exists():
insert_after = deliverables.index("reports/security_trust_report.md") + 1
deliverables.insert(insert_after, "reports/runtime_permission_probes.md")
if (skill_dir / "reports" / "output_blind_review_pack.md").exists():
insert_after = deliverables.index("reports/output_quality_scorecard.md") + 1
deliverables.insert(insert_after, "reports/output_blind_review_pack.md")
if (skill_dir / "reports" / "output_execution_runs.md").exists():
insert_after = deliverables.index("reports/output_quality_scorecard.md") + 1
deliverables.insert(insert_after, "reports/output_execution_runs.md")
if (skill_dir / "reports" / "output_blind_answer_key.json").exists():
insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1
deliverables.insert(insert_after, "reports/output_blind_answer_key.json")
if (skill_dir / "reports" / "output_review_adjudication.md").exists():
insert_after = deliverables.index("reports/output_blind_answer_key.json") + 1 if "reports/output_blind_answer_key.json" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1
deliverables.insert(insert_after, "reports/output_review_adjudication.md")
skill_summary = {
"name": name,
"title": title,
@@ -494,17 +560,7 @@ def build_report_model(skill_dir: Path) -> dict:
"updated_at": metadata["updated_at"],
"core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。",
"audience": "Skill 作者、复用团队和后续 reviewer。",
"deliverables": [
"SKILL.md",
"agents/interface.yaml",
"reports/skill-ir.json",
"reports/output_quality_scorecard.md",
"reports/conformance_matrix.md",
"reports/security_trust_report.md",
"reports/skill_atlas.html",
"reports/review-studio.html",
"reports/skill-overview.html",
],
"deliverables": deliverables,
"flow": ["输入材料", "Skill 包体", "可复用能力"],
}
contract = {
@@ -573,10 +629,115 @@ def build_report_model(skill_dir: Path) -> dict:
"trigger_samples": len(skill_ir.get("trigger_surface", {}).get("should_trigger", [])),
"output_eval_cases": len(skill_ir.get("eval_plan", {}).get("output", [])),
},
"compiled_targets": {
"ok": compiled_targets.get("ok", False),
"schema_version": compiled_targets.get("schema_version", ""),
"summary": compiled_targets.get("summary", {}),
"targets": [
{
"target": item.get("target", ""),
"status": item.get("status", ""),
"adapter_mode": item.get("target_transform", {}).get("adapter_mode", ""),
"degradation_strategy": item.get("target_transform", {}).get("degradation_strategy", ""),
"native_surface": item.get("target_native_contract", {}).get("native_surface", ""),
"permission_enforcement": item.get("target_native_contract", {}).get("permissions", {}).get("enforcement", ""),
"generated_files": item.get("target_transform", {}).get("generated_files", []),
"unsupported_features": item.get("unsupported_features", []),
"warnings": item.get("warnings", []),
}
for item in compiled_targets.get("targets", [])
if isinstance(item, dict)
],
"failures": compiled_targets.get("failures", []),
"warnings": compiled_targets.get("warnings", []),
},
"output_quality": output_quality.get("summary", {}),
"output_execution": {
"ok": output_execution.get("ok", False),
"summary": output_execution.get("summary", {}),
"runner": output_execution.get("runner", {}),
"failures": output_execution.get("failures", []),
},
"output_blind_review": {
"summary": output_blind_review.get("summary", {}),
"seed": output_blind_review.get("seed", ""),
"pair_count": output_blind_review.get("summary", {}).get("pair_count", 0),
"answer_key_separate": output_blind_review.get("summary", {}).get("answer_key_separate", False),
},
"output_review_adjudication": {
"ok": output_review_adjudication.get("ok", False),
"summary": output_review_adjudication.get("summary", {}),
"reviewer": output_review_adjudication.get("reviewer", ""),
"reviewed_at": output_review_adjudication.get("reviewed_at", ""),
"failures": output_review_adjudication.get("failures", []),
},
"runtime_conformance": conformance.get("summary", {}),
"runtime_permissions": {
"ok": runtime_permissions.get("ok", False),
"summary": runtime_permissions.get("summary", {}),
"expected_capabilities": runtime_permissions.get("expected_capabilities", []),
"targets": [
{
"target": item.get("target", ""),
"status": item.get("status", ""),
"assurance": item.get("assurance", ""),
"native_enforcement": item.get("native_enforcement"),
"metadata_fallback_explicit": item.get("metadata_fallback_explicit", False),
"residual_risks": item.get("residual_risks", []),
}
for item in runtime_permissions.get("targets", [])
if isinstance(item, dict)
],
"failures": runtime_permissions.get("failures", []),
},
"trust_security": trust_report.get("summary", {}),
"skill_atlas": skill_atlas.get("summary", {}),
"registry_distribution": {
"ok": registry_audit.get("ok", False),
"package": registry_audit.get("package", {}),
"failures": registry_audit.get("failures", []),
"warnings": registry_audit.get("warnings", []),
},
"package_verification": {
"ok": package_verification.get("ok", False),
"summary": package_verification.get("summary", {}),
"failures": package_verification.get("failures", []),
"warnings": package_verification.get("warnings", []),
},
"install_simulation": {
"ok": install_simulation.get("ok", False),
"summary": install_simulation.get("summary", {}),
"failures": install_simulation.get("failures", []),
"warnings": install_simulation.get("warnings", []),
},
"upgrade_check": {
"ok": upgrade_check.get("ok", False),
"summary": upgrade_check.get("summary", {}),
"upgrade_diff": upgrade_check.get("upgrade_diff", {}),
"release_notes": upgrade_check.get("release_notes", []),
"failures": upgrade_check.get("failures", []),
"warnings": upgrade_check.get("warnings", []),
},
"adoption_drift": {
"ok": adoption_drift.get("ok", False),
"summary": adoption_drift.get("summary", {}),
"next_iteration_candidates": adoption_drift.get("next_iteration_candidates", []),
"privacy_contract": adoption_drift.get("privacy_contract", {}),
"failures": adoption_drift.get("failures", []),
},
"review_waivers": {
"ok": review_waivers.get("ok", False),
"summary": review_waivers.get("summary", {}),
"policy": review_waivers.get("policy", {}),
"failures": review_waivers.get("failures", []),
"warnings": review_waivers.get("warnings", []),
},
"review_annotations": {
"ok": review_annotations.get("ok", False),
"summary": review_annotations.get("summary", {}),
"annotations": review_annotations.get("annotations", [])[:8],
"failures": review_annotations.get("failures", []),
},
"synthesis_highlights": synthesis,
"artifact_design": q_review["artifact_design"],
"prompt_quality": q_review["prompt_quality"],
scripts/trust_check.py
+449 -9
View File
@@ -3,9 +3,13 @@ import argparse
import ast
import hashlib
import json
import subprocess
import sys
import re
from datetime import date
from pathlib import Path
from typing import Any
from urllib.parse import urlparse
try:
import yaml
@@ -17,6 +21,13 @@ ROOT = Path(__file__).resolve().parent.parent
SCAN_DIRS = ["agents", "docs", "evals", "references", "runtime", "scripts", "security", "skill-ir", "templates"]
ROOT_FILES = ["SKILL.md", "README.md", "manifest.json", "requirements-ci.txt", "Makefile"]
TEXT_SUFFIXES = {".md", ".json", ".jsonl", ".yaml", ".yml", ".py", ".sh", ".txt", ".toml"}
PACKAGE_HASH_SCOPE = "source-contract-without-generated-reports"
INTERNAL_SCRIPT_INTERFACE = "internal-module"
NETWORK_POLICY_REL_PATH = "security/network_policy.json"
PERMISSION_POLICY_REL_PATH = "security/permission_policy.json"
HELP_SMOKE_TIMEOUT_SECONDS = 5.0
PERMISSION_CAPABILITIES = ("network", "file_write", "subprocess", "interactive")
PERMISSION_TARGETS = ("openai", "claude", "generic")
SECRET_PATTERNS = [
("private_key", re.compile(r"-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY-----")),
("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b")),
@@ -91,19 +102,80 @@ def script_inventory(skill_dir: Path) -> list[dict[str, Any]]:
for path in sorted(scripts_dir.glob("*.py")):
text = path.read_text(encoding="utf-8", errors="replace")
flags = script_flags(text)
interface = script_interface(text)
urls = extract_url_literals(text)
inventory.append(
{
"path": relpath(skill_dir, path),
"interface": interface["name"],
"interface_declared": interface["declared"],
"interface_reason": interface["reason"],
"has_argparse": "argparse" in text,
"has_main_guard": 'if __name__ == "__main__"' in text,
"uses_input": flags["uses_input"],
"uses_network": flags["uses_network"],
"uses_file_write": flags["uses_file_write"],
"uses_subprocess": flags["uses_subprocess"],
"network_urls": urls,
"network_hosts": sorted({urlparse(url).hostname or "" for url in urls if urlparse(url).hostname}),
}
)
return inventory
def string_assignment(tree: ast.Module, variable_name: str) -> str:
for node in tree.body:
value_node = None
if isinstance(node, ast.Assign):
if any(isinstance(target, ast.Name) and target.id == variable_name for target in node.targets):
value_node = node.value
elif isinstance(node, ast.AnnAssign):
target = node.target
if isinstance(target, ast.Name) and target.id == variable_name:
value_node = node.value
if isinstance(value_node, ast.Constant) and isinstance(value_node.value, str):
return value_node.value
return ""
def script_interface(text: str) -> dict[str, Any]:
try:
tree = ast.parse(text)
except SyntaxError:
match = re.search(r"SCRIPT_INTERFACE\s*=\s*['\"]([^'\"]+)['\"]", text)
reason_match = re.search(r"SCRIPT_INTERFACE_REASON\s*=\s*['\"]([^'\"]+)['\"]", text)
name = match.group(1) if match else "cli"
reason = reason_match.group(1) if reason_match else ""
return {"name": name, "declared": bool(match), "reason": reason}
name = string_assignment(tree, "SCRIPT_INTERFACE")
reason = string_assignment(tree, "SCRIPT_INTERFACE_REASON")
if name:
return {"name": name, "declared": True, "reason": reason}
return {"name": "cli", "declared": False, "reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules."}
def extract_url_literals(text: str) -> list[str]:
values: list[str] = []
try:
tree = ast.parse(text)
for node in ast.walk(tree):
if isinstance(node, ast.Constant) and isinstance(node.value, str):
values.append(node.value)
except SyntaxError:
values = re.findall(r"['\"]([^'\"]+)['\"]", text)
urls = []
seen = set()
for value in values:
for match in re.finditer(r"https?://[^\s'\"<>]+", value):
url = match.group(0).rstrip(").,]")
if url not in seen:
seen.add(url)
urls.append(url)
return urls
def call_name(node: ast.AST) -> str:
if isinstance(node, ast.Name):
return node.id
@@ -120,16 +192,39 @@ def script_flags(text: str) -> dict[str, bool]:
return {
"uses_input": bool(re.search(r"\b(?:input|getpass)\s*\(", text)),
"uses_network": bool(re.search(r"\b(?:urlopen|Request)\s*\(|\brequests\.", text)),
"uses_file_write": bool(
re.search(r"\.(?:write_text|write_bytes|mkdir|unlink)\s*\(", text)
or re.search(r"\b(?:open)\s*\([^)]*['\"][wa+x]", text)
or "shutil.rmtree" in text
),
"uses_subprocess": "subprocess." in text,
}
calls = [call_name(node.func) for node in ast.walk(tree) if isinstance(node, ast.Call)]
call_nodes = [node for node in ast.walk(tree) if isinstance(node, ast.Call)]
calls = [call_name(node.func) for node in call_nodes]
return {
"uses_input": any(name in {"input", "getpass.getpass"} or name.endswith(".getpass") for name in calls),
"uses_network": any(name in {"urlopen", "Request"} or name.startswith("requests.") for name in calls),
"uses_file_write": any(is_file_write_call(node, call_name(node.func)) for node in call_nodes),
"uses_subprocess": any(name.startswith("subprocess.") for name in calls),
}
def is_file_write_call(node: ast.Call, name: str) -> bool:
if name.endswith((".write_text", ".write_bytes", ".mkdir", ".unlink", ".rmdir")):
return True
if name in {"shutil.copy", "shutil.copy2", "shutil.copytree", "shutil.move", "shutil.rmtree"}:
return True
if name == "zipfile.ZipFile":
return True
if name in {"open", "Path.open"} or name.endswith(".open"):
args = list(node.args)
keywords = {keyword.arg: keyword.value for keyword in node.keywords if keyword.arg}
mode_node = args[1] if len(args) > 1 else keywords.get("mode")
if isinstance(mode_node, ast.Constant) and isinstance(mode_node.value, str):
return any(flag in mode_node.value for flag in ("w", "a", "x", "+"))
return False
def dependency_status(skill_dir: Path) -> dict[str, Any]:
candidates = ["requirements-ci.txt", "requirements.txt", "pyproject.toml", "package-lock.json", "uv.lock", "poetry.lock"]
present = [name for name in candidates if (skill_dir / name).exists()]
@@ -148,6 +243,179 @@ def dependency_status(skill_dir: Path) -> dict[str, Any]:
return {"present": present, "pinned": pinned, "unpinned": unpinned}
def script_capability_paths(scripts: list[dict[str, Any]]) -> dict[str, list[str]]:
return {
"network": [item["path"] for item in scripts if item["uses_network"]],
"file_write": [item["path"] for item in scripts if item.get("uses_file_write", False)],
"subprocess": [item["path"] for item in scripts if item["uses_subprocess"]],
"interactive": [item["path"] for item in scripts if item["uses_input"]],
}
def load_network_policy(skill_dir: Path) -> dict[str, Any]:
path = skill_dir / NETWORK_POLICY_REL_PATH
if not path.exists():
return {"present": False, "path": NETWORK_POLICY_REL_PATH, "scripts": {}}
payload = load_json(path)
scripts = payload.get("scripts", {})
return {
"present": True,
"path": NETWORK_POLICY_REL_PATH,
"schema_version": payload.get("schema_version", ""),
"default_policy": payload.get("default_policy", {}),
"scripts": scripts if isinstance(scripts, dict) else {},
}
def load_permission_policy(skill_dir: Path) -> dict[str, Any]:
path = skill_dir / PERMISSION_POLICY_REL_PATH
if not path.exists():
return {"present": False, "path": PERMISSION_POLICY_REL_PATH, "capabilities": {}}
payload = load_json(path)
capabilities = payload.get("capabilities", {})
return {
"present": True,
"path": PERMISSION_POLICY_REL_PATH,
"schema_version": payload.get("schema_version", ""),
"reviewed_at": payload.get("reviewed_at", ""),
"capabilities": capabilities if isinstance(capabilities, dict) else {},
}
def parse_iso_date(value: Any) -> date | None:
try:
return date.fromisoformat(str(value)[:10])
except (TypeError, ValueError):
return None
def permission_governance_status(skill_dir: Path, scripts: list[dict[str, Any]], today: date | None = None) -> dict[str, Any]:
policy = load_permission_policy(skill_dir)
today = today or date.today()
capability_paths = script_capability_paths(scripts)
required = {name: paths for name, paths in capability_paths.items() if paths}
approvals = []
missing = []
invalid = []
expired = []
approved = []
for name, paths in required.items():
entry = policy["capabilities"].get(name, {}) if policy["present"] else {}
if not isinstance(entry, dict) or not entry:
missing.append(name)
approvals.append({"capability": name, "status": "missing", "scripts": paths, "validation": ["approval entry is missing"]})
continue
validation = []
if entry.get("decision") != "approved":
validation.append("decision must be approved")
if not entry.get("reviewer"):
validation.append("reviewer is required")
if len(str(entry.get("reason", ""))) < 30:
validation.append("reason must be at least 30 characters")
if not entry.get("scope"):
validation.append("scope is required")
expires_at = parse_iso_date(entry.get("expires_at"))
if expires_at is None:
validation.append("expires_at must be ISO date")
target_enforcement = entry.get("target_enforcement", {})
if not isinstance(target_enforcement, dict):
validation.append("target_enforcement must be an object")
target_enforcement = {}
missing_targets = [target for target in PERMISSION_TARGETS if not target_enforcement.get(target)]
if missing_targets:
validation.append(f"target_enforcement missing: {', '.join(missing_targets)}")
if validation:
invalid.append(name)
status = "invalid"
elif expires_at is not None and expires_at < today:
expired.append(name)
status = "expired"
else:
approved.append(name)
status = "approved"
approvals.append(
{
"capability": name,
"status": status,
"scripts": paths,
"reviewer": str(entry.get("reviewer", "")),
"scope": str(entry.get("scope", "")),
"reason": str(entry.get("reason", "")),
"expires_at": str(entry.get("expires_at", "")),
"evidence": entry.get("evidence", []) if isinstance(entry.get("evidence", []), list) else [],
"target_enforcement": target_enforcement,
"validation": validation,
}
)
return {
"present": policy["present"],
"path": policy["path"],
"schema_version": policy.get("schema_version", ""),
"reviewed_at": policy.get("reviewed_at", ""),
"required_capabilities": sorted(required),
"approved_capabilities": sorted(approved),
"missing_capabilities": sorted(missing),
"invalid_capabilities": sorted(invalid),
"expired_capabilities": sorted(expired),
"approval_count": len(approved),
"required_count": len(required),
"missing_count": len(missing),
"invalid_count": len(invalid),
"expired_count": len(expired),
"approvals": approvals,
}
def network_policy_status(skill_dir: Path, scripts: list[dict[str, Any]]) -> dict[str, Any]:
policy = load_network_policy(skill_dir)
network_scripts = [item for item in scripts if item["uses_network"]]
covered = []
missing = []
mismatches = []
for item in network_scripts:
entry = policy["scripts"].get(item["path"], {}) if policy["present"] else {}
if not isinstance(entry, dict) or not entry:
missing.append(item["path"])
continue
allowed_hosts = set(entry.get("allowed_hosts") or [])
if not allowed_hosts:
mismatches.append(
{
"path": item["path"],
"reason": "allowed_hosts is empty",
"observed_hosts": item["network_hosts"],
"allowed_hosts": [],
}
)
continue
observed_hosts = set(item.get("network_hosts") or [])
unexpected_hosts = sorted(observed_hosts - allowed_hosts)
if unexpected_hosts:
mismatches.append(
{
"path": item["path"],
"reason": "observed HTTPS hosts are not declared in policy",
"observed_hosts": sorted(observed_hosts),
"allowed_hosts": sorted(allowed_hosts),
"unexpected_hosts": unexpected_hosts,
}
)
continue
covered.append(item["path"])
return {
"present": policy["present"],
"path": policy["path"],
"schema_version": policy.get("schema_version", ""),
"network_script_count": len(network_scripts),
"covered_scripts": covered,
"missing_scripts": missing,
"mismatches": mismatches,
"default_policy": policy.get("default_policy", {}),
}
def interface_trust(skill_dir: Path) -> dict[str, Any]:
interface = load_yaml(skill_dir / "agents" / "interface.yaml")
trust = interface.get("compatibility", {}).get("trust", {})
@@ -169,12 +437,106 @@ def package_digest(files: list[Path], skill_dir: Path) -> str:
return digest.hexdigest()
def build_trust_report(skill_dir: Path) -> dict[str, Any]:
def help_smoke_candidates(scripts: list[dict[str, Any]]) -> list[dict[str, Any]]:
return [
item
for item in scripts
if item["interface"] != INTERNAL_SCRIPT_INTERFACE and item["has_argparse"]
]
def run_help_smoke_checks(skill_dir: Path, scripts: list[dict[str, Any]], timeout: float) -> dict[str, Any]:
candidates = help_smoke_candidates(scripts)
skipped = [
{
"path": item["path"],
"reason": "internal module" if item["interface"] == INTERNAL_SCRIPT_INTERFACE else "missing argparse/help surface",
}
for item in scripts
if item not in candidates
]
results = []
for item in candidates:
command = [sys.executable, item["path"], "--help"]
try:
proc = subprocess.run(
command,
cwd=skill_dir,
capture_output=True,
text=True,
timeout=timeout,
)
output = f"{proc.stdout}\n{proc.stderr}".lower()
has_help_text = "usage:" in output or "--help" in output
results.append(
{
"path": item["path"],
"command": f"python3 {item['path']} --help",
"returncode": proc.returncode,
"timed_out": False,
"passed": proc.returncode == 0 and has_help_text,
"has_help_text": has_help_text,
"stdout_excerpt": proc.stdout.strip()[:240],
"stderr_excerpt": proc.stderr.strip()[:240],
}
)
except subprocess.TimeoutExpired as exc:
results.append(
{
"path": item["path"],
"command": f"python3 {item['path']} --help",
"returncode": None,
"timed_out": True,
"passed": False,
"has_help_text": False,
"stdout_excerpt": (exc.stdout or "").strip()[:240] if isinstance(exc.stdout, str) else "",
"stderr_excerpt": (exc.stderr or "").strip()[:240] if isinstance(exc.stderr, str) else "",
}
)
failed = [item for item in results if not item["passed"]]
return {
"enabled": True,
"timeout_seconds": timeout,
"candidate_count": len(candidates),
"checked_count": len(results),
"passed_count": len(results) - len(failed),
"failed_count": len(failed),
"skipped_count": len(skipped),
"failed_scripts": [item["path"] for item in failed],
"results": results,
"skipped": skipped,
}
def disabled_help_smoke_status(scripts: list[dict[str, Any]], timeout: float) -> dict[str, Any]:
candidates = help_smoke_candidates(scripts)
return {
"enabled": False,
"timeout_seconds": timeout,
"candidate_count": len(candidates),
"checked_count": 0,
"passed_count": 0,
"failed_count": 0,
"skipped_count": len(scripts),
"failed_scripts": [],
"results": [],
"skipped": [{"path": item["path"], "reason": "help smoke disabled"} for item in scripts],
}
def build_trust_report(skill_dir: Path, run_help_smoke: bool = True, help_smoke_timeout: float = HELP_SMOKE_TIMEOUT_SECONDS) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
files = iter_scan_files(skill_dir)
secrets = scan_secrets(skill_dir, files)
scripts = script_inventory(skill_dir)
deps = dependency_status(skill_dir)
network_policy = network_policy_status(skill_dir, scripts)
help_smoke = (
run_help_smoke_checks(skill_dir, scripts, help_smoke_timeout)
if run_help_smoke
else disabled_help_smoke_status(scripts, help_smoke_timeout)
)
permission_governance = permission_governance_status(skill_dir, scripts)
trust = interface_trust(skill_dir)
failures = []
warnings = []
@@ -187,23 +549,51 @@ def build_trust_report(skill_dir: Path) -> dict[str, Any]:
warnings.append(f"Unpinned dependency entries: {', '.join(deps['unpinned'])}")
if not deps["present"]:
warnings.append("No dependency or lock file detected")
missing_help = [item["path"] for item in scripts if not item["has_argparse"]]
internal_modules = [item for item in scripts if item["interface"] == INTERNAL_SCRIPT_INTERFACE]
missing_help = [
item["path"] for item in scripts if item["interface"] != INTERNAL_SCRIPT_INTERFACE and not item["has_argparse"]
]
if missing_help:
warnings.append(f"Scripts without argparse/help surface: {', '.join(missing_help[:8])}")
warnings.append(f"CLI scripts without argparse/help surface: {', '.join(missing_help[:8])}")
interactive = [item["path"] for item in scripts if item["uses_input"]]
if interactive:
warnings.append(f"Interactive scripts require reviewer awareness: {', '.join(interactive[:8])}")
network = [item["path"] for item in scripts if item["uses_network"]]
if network:
warnings.append(f"Network-capable scripts require bounded host policy: {', '.join(network[:8])}")
file_write = [item["path"] for item in scripts if item["uses_file_write"]]
if network_policy["missing_scripts"]:
warnings.append(f"Network-capable scripts require bounded host policy: {', '.join(network_policy['missing_scripts'][:8])}")
if network_policy["mismatches"]:
warning_paths = [item["path"] for item in network_policy["mismatches"]]
warnings.append(f"Network host policy mismatch: {', '.join(warning_paths[:8])}")
if help_smoke["failed_scripts"]:
warnings.append(f"CLI help smoke failed: {', '.join(help_smoke['failed_scripts'][:8])}")
if permission_governance["missing_capabilities"]:
warnings.append(f"Permission approvals missing: {', '.join(permission_governance['missing_capabilities'][:8])}")
if permission_governance["invalid_capabilities"]:
warnings.append(f"Permission approvals invalid: {', '.join(permission_governance['invalid_capabilities'][:8])}")
if permission_governance["expired_capabilities"]:
warnings.append(f"Permission approvals expired: {', '.join(permission_governance['expired_capabilities'][:8])}")
summary = {
"scanned_files": len(files),
"script_count": len(scripts),
"internal_module_count": len(internal_modules),
"secret_findings": len(secrets),
"dependency_files": deps["present"],
"network_script_count": len(network),
"network_policy_covered_count": len(network_policy["covered_scripts"]),
"network_policy_missing_count": len(network_policy["missing_scripts"]),
"file_write_script_count": len(file_write),
"permission_required_count": permission_governance["required_count"],
"permission_approved_count": permission_governance["approval_count"],
"permission_missing_count": permission_governance["missing_count"],
"permission_invalid_count": permission_governance["invalid_count"],
"permission_expired_count": permission_governance["expired_count"],
"help_smoke_checked_count": help_smoke["checked_count"],
"help_smoke_failed_count": help_smoke["failed_count"],
"interactive_script_count": len(interactive),
"package_hash_scope": PACKAGE_HASH_SCOPE,
"package_hash_file_count": len(files),
"package_sha256": package_digest(files, skill_dir),
}
return {
@@ -215,6 +605,9 @@ def build_trust_report(skill_dir: Path) -> dict[str, Any]:
"secrets": secrets,
"scripts": scripts,
"dependencies": deps,
"network_policy": network_policy,
"help_smoke": help_smoke,
"permission_governance": permission_governance,
"trust_metadata": trust,
}
@@ -227,9 +620,19 @@ def render_markdown(payload: dict[str, Any]) -> str:
f"- OK: `{payload['ok']}`",
f"- Scanned files: `{summary['scanned_files']}`",
f"- Scripts: `{summary['script_count']}`",
f"- Internal script modules: `{summary.get('internal_module_count', 0)}`",
f"- Secret findings: `{summary['secret_findings']}`",
f"- Network-capable scripts: `{summary['network_script_count']}`",
f"- Network policy covered scripts: `{summary.get('network_policy_covered_count', 0)}`",
f"- Network policy missing scripts: `{summary.get('network_policy_missing_count', 0)}`",
f"- File-write scripts: `{summary.get('file_write_script_count', 0)}`",
f"- Permission approvals: `{summary.get('permission_approved_count', 0)} / {summary.get('permission_required_count', 0)}`",
f"- Permission approval gaps: `{summary.get('permission_missing_count', 0) + summary.get('permission_invalid_count', 0) + summary.get('permission_expired_count', 0)}`",
f"- CLI help smoke checked: `{summary.get('help_smoke_checked_count', 0)}`",
f"- CLI help smoke failures: `{summary.get('help_smoke_failed_count', 0)}`",
f"- Interactive scripts: `{summary['interactive_script_count']}`",
f"- Package hash scope: `{summary['package_hash_scope']}`",
f"- Package hash files: `{summary['package_hash_file_count']}`",
f"- Package SHA256: `{summary['package_sha256']}`",
"",
"## Failures",
@@ -242,10 +645,41 @@ def render_markdown(payload: dict[str, Any]) -> str:
lines.append(f"- Files: `{', '.join(payload['dependencies']['present']) or 'none'}`")
lines.append(f"- Pinned entries: `{len(payload['dependencies']['pinned'])}`")
lines.append(f"- Unpinned entries: `{len(payload['dependencies']['unpinned'])}`")
lines.extend(["", "## Script Surface", "", "| Script | Argparse | Main Guard | Input | Network | Subprocess |", "| --- | --- | --- | --- | --- | --- |"])
network_policy = payload.get("network_policy", {})
lines.extend(["", "## Network Policy", ""])
lines.append(f"- Policy file: `{network_policy.get('path', NETWORK_POLICY_REL_PATH)}`")
lines.append(f"- Present: `{network_policy.get('present', False)}`")
lines.append(f"- Covered scripts: `{len(network_policy.get('covered_scripts', []))}`")
lines.append(f"- Missing scripts: `{', '.join(network_policy.get('missing_scripts', [])) or 'none'}`")
lines.append(f"- Mismatches: `{len(network_policy.get('mismatches', []))}`")
permission_governance = payload.get("permission_governance", {})
lines.extend(["", "## Permission Governance", ""])
lines.append(f"- Policy file: `{permission_governance.get('path', PERMISSION_POLICY_REL_PATH)}`")
lines.append(f"- Present: `{permission_governance.get('present', False)}`")
lines.append(f"- Required capabilities: `{', '.join(permission_governance.get('required_capabilities', [])) or 'none'}`")
lines.append(f"- Approved capabilities: `{', '.join(permission_governance.get('approved_capabilities', [])) or 'none'}`")
lines.append(f"- Missing approvals: `{', '.join(permission_governance.get('missing_capabilities', [])) or 'none'}`")
lines.append(f"- Invalid approvals: `{', '.join(permission_governance.get('invalid_capabilities', [])) or 'none'}`")
lines.append(f"- Expired approvals: `{', '.join(permission_governance.get('expired_capabilities', [])) or 'none'}`")
help_smoke = payload.get("help_smoke", {})
lines.extend(["", "## CLI Help Smoke", ""])
lines.append(f"- Enabled: `{help_smoke.get('enabled', False)}`")
lines.append(f"- Timeout seconds: `{help_smoke.get('timeout_seconds', HELP_SMOKE_TIMEOUT_SECONDS)}`")
lines.append(f"- Checked scripts: `{help_smoke.get('checked_count', 0)}`")
lines.append(f"- Passed scripts: `{help_smoke.get('passed_count', 0)}`")
lines.append(f"- Failed scripts: `{', '.join(help_smoke.get('failed_scripts', [])) or 'none'}`")
lines.extend(
[
"",
"## Script Surface",
"",
"| Script | Interface | Declared | Argparse | Main Guard | Input | Network | File Write | Subprocess | Reason |",
"| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |",
]
)
for item in payload["scripts"]:
lines.append(
f"| {item['path']} | {item['has_argparse']} | {item['has_main_guard']} | {item['uses_input']} | {item['uses_network']} | {item['uses_subprocess']} |"
f"| {item['path']} | {item['interface']} | {item['interface_declared']} | {item['has_argparse']} | {item['has_main_guard']} | {item['uses_input']} | {item['uses_network']} | {item.get('uses_file_write', False)} | {item['uses_subprocess']} | {item['interface_reason']} |"
)
return "\n".join(lines).strip() + "\n"
@@ -255,12 +689,18 @@ def main() -> None:
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--output-json")
parser.add_argument("--output-md")
parser.add_argument("--skip-help-smoke", action="store_true")
parser.add_argument("--help-smoke-timeout", type=float, default=HELP_SMOKE_TIMEOUT_SECONDS)
args = parser.parse_args()
skill_dir = Path(args.skill_dir).resolve()
output_json = Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "security_trust_report.json"
output_md = Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "security_trust_report.md"
payload = build_trust_report(skill_dir)
payload = build_trust_report(
skill_dir,
run_help_smoke=not args.skip_help_smoke,
help_smoke_timeout=args.help_smoke_timeout,
)
payload["artifacts"] = {"json": display_path(output_json), "markdown": display_path(output_md)}
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
scripts/upgrade_check.py
+268
View File
@@ -0,0 +1,268 @@
#!/usr/bin/env python3
import argparse
import json
import re
from datetime import date
from pathlib import Path
from typing import Any
ROOT = Path(__file__).resolve().parent.parent
BUMP_RANK = {"none": 0, "patch": 1, "minor": 2, "major": 3}
COMPAT_RANK = {"missing": 0, "unknown": 0, "fail": 0, "warn": 1, "pass": 2}
def display_path(path: Path, root: Path = ROOT) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def package_from_payload(payload: dict[str, Any]) -> dict[str, Any]:
package = payload.get("package")
if isinstance(package, dict):
return package
return payload
def parse_version(value: str) -> tuple[int, int, int] | None:
match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$", str(value or ""))
if not match:
return None
return tuple(int(part) for part in match.groups())
def declared_bump(previous: str, current: str) -> str:
old = parse_version(previous)
new = parse_version(current)
if old is None or new is None:
return "invalid"
if new < old:
return "downgrade"
if new == old:
return "none"
if new[0] > old[0]:
return "major"
if new[1] > old[1]:
return "minor"
return "patch"
def compare_sets(previous: list[Any], current: list[Any]) -> tuple[list[str], list[str]]:
old = {str(item) for item in previous}
new = {str(item) for item in current}
return sorted(new - old), sorted(old - new)
def compatibility_changes(previous: dict[str, Any], current: dict[str, Any]) -> list[dict[str, str]]:
targets = sorted(set(previous) | set(current))
changes = []
for target in targets:
old = str(previous.get(target, "missing"))
new = str(current.get(target, "missing"))
if old != new:
direction = "regressed" if COMPAT_RANK.get(new, 0) < COMPAT_RANK.get(old, 0) else "improved"
changes.append({"target": target, "from": old, "to": new, "direction": direction})
return changes
def metadata_changes(previous: dict[str, Any], current: dict[str, Any]) -> list[dict[str, str]]:
fields = ["description", "maturity", "owner", "review_cadence", "trust_level", "license"]
changes = []
for field in fields:
old = str(previous.get(field, ""))
new = str(current.get(field, ""))
if old != new:
changes.append({"field": field, "from": old, "to": new})
return changes
def checksum_changes(previous: dict[str, Any], current: dict[str, Any]) -> list[dict[str, str]]:
old_checksums = previous.get("checksums", {}) if isinstance(previous.get("checksums"), dict) else {}
new_checksums = current.get("checksums", {}) if isinstance(current.get("checksums"), dict) else {}
changes = []
for key in sorted(set(old_checksums) | set(new_checksums)):
old = str(old_checksums.get(key, ""))
new = str(new_checksums.get(key, ""))
if old != new:
changes.append({"field": key, "from": old, "to": new})
return changes
def required_bump(
*,
name_changed: bool,
removed_targets: list[str],
added_targets: list[str],
compat: list[dict[str, str]],
metadata: list[dict[str, str]],
checksums: list[dict[str, str]],
) -> str:
if name_changed or removed_targets or any(item["direction"] == "regressed" for item in compat):
return "major"
if added_targets or any(item["field"] in {"description", "maturity", "trust_level"} for item in metadata):
return "minor"
if metadata or checksums or compat:
return "patch"
return "none"
def build_release_notes(diff: dict[str, Any], recommended: str, current: dict[str, Any]) -> list[str]:
notes = [f"Recommended version bump: {recommended}."]
if diff["removed_targets"]:
notes.append(f"Breaking target removals: {', '.join(diff['removed_targets'])}.")
if diff["added_targets"]:
notes.append(f"Added targets: {', '.join(diff['added_targets'])}.")
regressions = [item for item in diff["compatibility_changes"] if item["direction"] == "regressed"]
if regressions:
notes.append("Compatibility regressions: " + ", ".join(f"{item['target']} {item['from']}->{item['to']}" for item in regressions) + ".")
if diff["checksum_changes"]:
notes.append("Package or archive checksum changed; reviewers should verify package artifacts before release.")
if current.get("artifacts", {}).get("package_verification"):
notes.append(f"Package verification evidence: {current['artifacts']['package_verification']}.")
return notes
def run_upgrade_check(previous: dict[str, Any], current: dict[str, Any], generated_at: str) -> dict[str, Any]:
failures = []
warnings = []
previous_version = str(previous.get("version", ""))
current_version = str(current.get("version", ""))
bump = declared_bump(previous_version, current_version)
if bump == "invalid":
failures.append("Version comparison requires semver-like previous and current versions")
if bump == "downgrade":
failures.append("Current version is lower than previous version")
added_targets, removed_targets = compare_sets(previous.get("targets", []), current.get("targets", []))
compat = compatibility_changes(previous.get("compatibility", {}), current.get("compatibility", {}))
metadata = metadata_changes(previous, current)
checksums = checksum_changes(previous, current)
name_changed = str(previous.get("name", "")) != str(current.get("name", ""))
recommended = required_bump(
name_changed=name_changed,
removed_targets=removed_targets,
added_targets=added_targets,
compat=compat,
metadata=metadata,
checksums=checksums,
)
if bump in BUMP_RANK and BUMP_RANK[bump] < BUMP_RANK[recommended]:
failures.append(f"Version bump is insufficient: declared {bump}, recommended {recommended}")
if name_changed:
failures.append("Package name changed; publish as a new package or use a major version bump")
if not checksums:
warnings.append("No checksum changes detected; confirm the previous package is the intended comparison baseline")
diff = {
"name_changed": name_changed,
"added_targets": added_targets,
"removed_targets": removed_targets,
"compatibility_changes": compat,
"metadata_changes": metadata,
"checksum_changes": checksums,
}
return {
"ok": not failures,
"schema_version": "2.0",
"generated_at": generated_at,
"previous": {
"name": previous.get("name", ""),
"version": previous_version,
"targets": previous.get("targets", []),
},
"current": {
"name": current.get("name", ""),
"version": current_version,
"targets": current.get("targets", []),
},
"summary": {
"declared_bump": bump,
"recommended_bump": recommended,
"breaking_change_count": int(name_changed) + len(removed_targets) + sum(1 for item in compat if item["direction"] == "regressed"),
"added_target_count": len(added_targets),
"metadata_change_count": len(metadata),
"checksum_change_count": len(checksums),
"failure_count": len(failures),
"warning_count": len(warnings),
},
"upgrade_diff": diff,
"release_notes": build_release_notes(diff, recommended, current),
"failures": failures,
"warnings": warnings,
}
def render_markdown(report: dict[str, Any]) -> str:
lines = [
"# Upgrade Check",
"",
f"- OK: `{report['ok']}`",
f"- Previous: `{report['previous']['name']} {report['previous']['version']}`",
f"- Current: `{report['current']['name']} {report['current']['version']}`",
f"- Declared bump: `{report['summary']['declared_bump']}`",
f"- Recommended bump: `{report['summary']['recommended_bump']}`",
f"- Breaking changes: `{report['summary']['breaking_change_count']}`",
"",
"## Upgrade Diff",
"",
f"- Added targets: `{', '.join(report['upgrade_diff']['added_targets']) or 'none'}`",
f"- Removed targets: `{', '.join(report['upgrade_diff']['removed_targets']) or 'none'}`",
f"- Compatibility changes: `{len(report['upgrade_diff']['compatibility_changes'])}`",
f"- Metadata changes: `{len(report['upgrade_diff']['metadata_changes'])}`",
f"- Checksum changes: `{len(report['upgrade_diff']['checksum_changes'])}`",
"",
"## Release Notes",
"",
]
lines.extend(f"- {item}" for item in report["release_notes"])
lines.extend(["", "## Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
lines.extend(["", "## Warnings", ""])
lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"])
return "\n".join(lines) + "\n"
def main() -> None:
parser = argparse.ArgumentParser(description="Compare current and previous registry package metadata for upgrade readiness.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--previous-package-json", required=True)
parser.add_argument("--current-package-json", default="reports/registry_audit.json")
parser.add_argument("--output-json", default="reports/upgrade_check.json")
parser.add_argument("--output-md", default="reports/upgrade_check.md")
parser.add_argument("--generated-at", default=str(date.today()))
args = parser.parse_args()
previous = package_from_payload(load_json(Path(args.previous_package_json)))
current = package_from_payload(load_json(Path(args.current_package_json)))
report = run_upgrade_check(previous, current, args.generated_at)
report["artifacts"] = {
"previous_package": display_path(Path(args.previous_package_json)),
"current_package": display_path(Path(args.current_package_json)),
}
output_json = Path(args.output_json)
output_md = Path(args.output_md)
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
report["artifacts"]["json"] = display_path(output_json)
report["artifacts"]["markdown"] = display_path(output_md)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
raise SystemExit(0 if report["ok"] else 2)
if __name__ == "__main__":
main()
scripts/verify_package.py
+261
View File
@@ -0,0 +1,261 @@
#!/usr/bin/env python3
import argparse
import hashlib
import json
import zipfile
from datetime import date
from pathlib import Path, PurePosixPath
from typing import Any
ROOT = Path(__file__).resolve().parent.parent
def display_path(path: Path, root: Path = ROOT) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def zip_names(zip_path: Path) -> list[str]:
with zipfile.ZipFile(zip_path) as archive:
return archive.namelist()
def unsafe_zip_entries(names: list[str]) -> list[str]:
unsafe = []
for name in names:
pure = PurePosixPath(name)
if pure.is_absolute() or ".." in pure.parts or name.startswith("\\") or (pure.parts and ":" in pure.parts[0]):
unsafe.append(name)
return unsafe
def generated_zip_entries(names: list[str]) -> list[str]:
generated = []
for name in names:
parts = PurePosixPath(name).parts
if "dist" in parts or (len(parts) > 2 and parts[1] == "tests" and any(part.startswith("tmp") for part in parts[2:])):
generated.append(name)
return generated
def required_targets(expectations: dict[str, Any], package_dir: Path) -> list[str]:
targets = expectations.get("required_targets") or []
if targets:
return [str(item) for item in targets]
targets_dir = package_dir / "targets"
if not targets_dir.exists():
return []
return sorted(path.name for path in targets_dir.iterdir() if path.is_dir())
def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str, passed: bool, detail: str) -> None:
checks.append({"id": check_id, "status": "pass" if passed else "fail", "detail": detail})
if not passed:
failures.append(detail)
def verify_package(
skill_dir: Path,
package_dir: Path,
expectations: dict[str, Any],
registry: dict[str, Any],
require_zip: bool,
generated_at: str,
) -> dict[str, Any]:
skill_dir = skill_dir.resolve()
package_dir = package_dir.resolve()
checks: list[dict[str, str]] = []
failures: list[str] = []
warnings: list[str] = []
manifest_path = package_dir / "manifest.json"
manifest = load_json(manifest_path)
add_check(checks, failures, "package-manifest", bool(manifest), f"Package manifest exists: {display_path(manifest_path)}")
targets = required_targets(expectations, package_dir)
adapter_paths = []
for target in targets:
adapter_path = package_dir / "targets" / target / "adapter.json"
adapter_paths.append(adapter_path)
adapter = load_json(adapter_path)
add_check(checks, failures, f"{target}-adapter", bool(adapter), f"Adapter exists for target: {target}")
for field in expectations.get("required_fields", []):
add_check(
checks,
failures,
f"{target}-field-{field}",
field in adapter,
f"{target} adapter includes field: {field}",
)
for target, key in (
("openai", "openai_required_files"),
("claude", "claude_required_files"),
("generic", "generic_required_files"),
):
for rel in expectations.get(key, []):
add_check(checks, failures, f"{target}-file-{rel}", (package_dir / rel).exists(), f"Package contains {rel}")
archive_path = package_dir / f"{skill_dir.name}.zip"
archive_sha = ""
archive_entries: list[str] = []
if archive_path.exists():
archive_sha = sha256_file(archive_path)
try:
archive_entries = zip_names(archive_path)
except zipfile.BadZipFile:
add_check(checks, failures, "archive-readable", False, f"Archive is not a readable zip: {display_path(archive_path)}")
else:
unsafe_entries = unsafe_zip_entries(archive_entries)
required_entries = [
f"{skill_dir.name}/SKILL.md",
f"{skill_dir.name}/manifest.json",
f"{skill_dir.name}/agents/interface.yaml",
]
add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries")
for entry in required_entries:
add_check(checks, failures, f"archive-entry-{entry}", entry in archive_entries, f"Archive contains {entry}")
generated_entries = generated_zip_entries(archive_entries)
add_check(checks, failures, "archive-excludes-generated", not generated_entries, "Archive excludes generated dist/ and tests/tmp* contents")
elif require_zip:
add_check(checks, failures, "archive-present", False, f"Missing required package archive: {display_path(archive_path)}")
else:
warnings.append(f"Package archive not found: {display_path(archive_path)}")
registry_package = registry.get("package", {}) if registry else {}
if registry_package:
add_check(checks, failures, "registry-ok", bool(registry.get("ok")), "Registry audit is OK")
add_check(
checks,
failures,
"registry-name-match",
registry_package.get("name") == manifest.get("name"),
"Registry package name matches package manifest",
)
add_check(
checks,
failures,
"registry-version-match",
registry_package.get("version") == manifest.get("version"),
"Registry package version matches package manifest",
)
compatibility = registry_package.get("compatibility", {})
for target in targets:
add_check(
checks,
failures,
f"registry-compat-{target}",
compatibility.get(target) in {"pass", "warn"},
f"Registry compatibility is reviewable for target: {target}",
)
else:
warnings.append("Registry audit was not supplied; package verification skipped metadata parity checks.")
report = {
"ok": not failures,
"schema_version": "2.0",
"generated_at": generated_at,
"skill_dir": display_path(skill_dir),
"package_dir": display_path(package_dir),
"summary": {
"target_count": len(targets),
"adapter_count": sum(1 for path in adapter_paths if path.exists()),
"archive_present": archive_path.exists(),
"archive_sha256": archive_sha,
"archive_entry_count": len(archive_entries),
"failure_count": len(failures),
"warning_count": len(warnings),
},
"checks": checks,
"failures": failures,
"warnings": warnings,
"artifacts": {
"manifest": display_path(manifest_path),
"archive": display_path(archive_path) if archive_path.exists() else "",
},
}
return report
def render_markdown(report: dict[str, Any]) -> str:
summary = report["summary"]
lines = [
"# Package Verification",
"",
f"- OK: `{report['ok']}`",
f"- Package directory: `{report['package_dir']}`",
f"- Targets: `{summary['adapter_count']} / {summary['target_count']}` adapters present",
f"- Archive present: `{summary['archive_present']}`",
f"- Archive SHA256: `{summary['archive_sha256'] or 'n/a'}`",
f"- Failures: `{summary['failure_count']}`",
f"- Warnings: `{summary['warning_count']}`",
"",
"## Checks",
"",
"| Check | Status | Detail |",
"| --- | --- | --- |",
]
for item in report["checks"]:
lines.append(f"| `{item['id']}` | `{item['status']}` | {item['detail']} |")
lines.extend(["", "## Failures", ""])
lines.extend([f"- {item}" for item in report["failures"]] or ["- None"])
lines.extend(["", "## Warnings", ""])
lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"])
return "\n".join(lines) + "\n"
def main() -> None:
parser = argparse.ArgumentParser(description="Verify generated skill package artifacts and archive integrity.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--package-dir", default="dist")
parser.add_argument("--expectations", default="evals/packaging_expectations.json")
parser.add_argument("--registry-json", default="reports/registry_audit.json")
parser.add_argument("--output-json", default="reports/package_verification.json")
parser.add_argument("--output-md", default="reports/package_verification.md")
parser.add_argument("--require-zip", action="store_true")
parser.add_argument("--generated-at", default=str(date.today()))
args = parser.parse_args()
skill_dir = Path(args.skill_dir)
package_dir = Path(args.package_dir)
if not package_dir.is_absolute():
package_dir = Path.cwd() / package_dir
expectations = load_json(Path(args.expectations)) if args.expectations else {}
registry = load_json(Path(args.registry_json)) if args.registry_json else {}
report = verify_package(skill_dir, package_dir, expectations, registry, args.require_zip, args.generated_at)
output_json = Path(args.output_json)
output_md = Path(args.output_md)
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(report), encoding="utf-8")
report["artifacts"]["json"] = display_path(output_json)
report["artifacts"]["markdown"] = display_path(output_md)
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
print(json.dumps(report, ensure_ascii=False, indent=2))
raise SystemExit(0 if report["ok"] else 2)
if __name__ == "__main__":
main()

Some files were not shown because too many files have changed in this diff Show More