From 3fb08802e96772df6bdae958f6b936c26de4b809 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Sat, 13 Jun 2026 18:00:32 +0800 Subject: [PATCH] Upgrade meta skill governance reports --- .gitignore | 1 + Makefile | 39 +- README.md | 65 +- SKILL.md | 4 +- evals/README.md | 28 +- evals/output/schema.json | 22 +- evals/packaging_expectations.json | 6 + references/distribution-registry-method.md | 57 + references/output-eval-method.md | 53 + references/packaging-contracts.md | 23 +- references/platform-capability-matrix.md | 25 +- references/review-studio-method.md | 64 +- references/review-waiver-method.md | 57 + references/skill-atlas-method.md | 8 +- references/telemetry-drift-method.md | 60 + references/trust-security-method.md | 54 +- registry/examples/yao-meta-skill-1.0.0.json | 43 + registry/index.json | 21 + registry/index.schema.json | 30 + registry/package.schema.json | 53 + registry/packages/yao-meta-skill.json | 56 + reports/adoption_drift_report.json | 76 + reports/adoption_drift_report.md | 35 + reports/artifact-design-profile.json | 16 +- reports/artifact-design-profile.md | 11 +- reports/compiled_targets.json | 3130 ++++++++ reports/compiled_targets.md | 63 + reports/conformance_matrix.json | 24 +- reports/conformance_matrix.md | 4 +- reports/context_budget.json | 12 +- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 6 +- reports/install_simulation.json | 121 + reports/install_simulation.md | 42 + reports/intent-confidence.json | 118 +- reports/intent-confidence.md | 27 +- reports/intent-context.json | 42 +- reports/output_blind_answer_key.json | 56 + reports/output_blind_review_pack.json | 204 + reports/output_blind_review_pack.md | 97 + reports/output_execution_runs.json | 291 + reports/output_execution_runs.md | 41 + reports/output_quality_scorecard.json | 13 +- reports/output_quality_scorecard.md | 4 + reports/output_review_adjudication.json | 73 + reports/output_review_adjudication.md | 29 + reports/package_verification.json | 516 ++ reports/package_verification.md | 120 + reports/prompt-quality-profile.json | 62 +- reports/prompt-quality-profile.md | 49 +- reports/reference-synthesis.json | 153 +- reports/reference-synthesis.md | 44 +- reports/registry_audit.json | 91 + reports/registry_audit.md | 37 + reports/review-studio.html | 173 +- reports/review-studio.json | 7407 ++++++++++++++++++- reports/review_annotations.json | 24 + reports/review_annotations.md | 19 + reports/review_waivers.json | 40 + reports/review_waivers.md | 26 + reports/runtime_permission_probes.json | 253 + reports/runtime_permission_probes.md | 27 + reports/security_trust_report.json | 1387 +++- reports/security_trust_report.md | 161 +- reports/skill-os-2-review.md | 106 +- reports/skill-overview.html | 45 +- reports/skill-overview.json | 608 +- reports/skill_atlas.html | 19 +- reports/skill_atlas.json | 218 +- reports/system-model.json | 53 +- reports/system-model.md | 46 +- reports/upgrade_check.json | 94 + reports/upgrade_check.md | 31 + scripts/adjudicate_output_review.py | 351 + scripts/build_skill_atlas.py | 127 +- scripts/ci_test.py | 11 + scripts/compile_skill.py | 726 ++ scripts/cross_packager.py | 61 +- scripts/export_skill_ir.py | 11 + scripts/init_skill.py | 43 +- scripts/local_output_eval_runner.py | 65 + scripts/probe_runtime_permissions.py | 289 + scripts/registry_audit.py | 377 + scripts/render_adoption_drift_report.py | 441 ++ scripts/render_context_reports.py | 14 +- scripts/render_review_annotations.py | 399 + scripts/render_review_studio.py | 842 ++- scripts/render_review_waivers.py | 281 + scripts/render_skill_overview.py | 54 + scripts/render_social_preview.py | 21 +- scripts/run_conformance_suite.py | 2 +- scripts/run_output_eval.py | 187 +- scripts/run_output_execution.py | 382 + scripts/simulate_install.py | 271 + scripts/skill_report_charts.py | 3 + scripts/skill_report_metrics.py | 10 + scripts/skill_report_model.py | 183 +- scripts/trust_check.py | 458 +- scripts/upgrade_check.py | 268 + scripts/verify_package.py | 261 + scripts/yao.py | 474 ++ security/network_policy.json | 37 + security/network_policy.md | 12 + security/permission_policy.json | 55 + security/permission_policy.md | 31 + security/script_policy.md | 2 + security/trust_policy.md | 7 + skill-ir/examples/yao-meta-skill.json | 32 +- skill_atlas/catalog.json | 82 +- skill_atlas/owner_review_gaps.json | 36 +- skill_atlas/policy.json | 23 + skill_atlas/route_overlap_matrix.csv | 134 +- skill_atlas/stale_skills.json | 40 +- tests/snapshots/claude_adapter.json | 8 +- tests/snapshots/generic_adapter.json | 8 +- tests/snapshots/openai_adapter.json | 8 +- tests/verify_adapter_snapshots.py | 69 + tests/verify_adoption_drift.py | 197 + tests/verify_compile_skill.py | 105 + tests/verify_install_simulation.py | 106 + tests/verify_output_eval_lab.py | 33 + tests/verify_output_execution_runs.py | 157 + tests/verify_output_review_adjudication.py | 185 + tests/verify_package_verification.py | 108 + tests/verify_registry_audit.py | 106 + tests/verify_review_annotations.py | 197 + tests/verify_review_studio.py | 313 +- tests/verify_review_waivers.py | 101 + tests/verify_runtime_permission_probes.py | 105 + tests/verify_skill_atlas.py | 25 + tests/verify_skill_overview.py | 35 + tests/verify_trust_check.py | 169 + tests/verify_upgrade_check.py | 92 + tests/verify_yao_cli.py | 243 +- 134 files changed, 25466 insertions(+), 1122 deletions(-) create mode 100644 references/distribution-registry-method.md create mode 100644 references/review-waiver-method.md create mode 100644 references/telemetry-drift-method.md create mode 100644 registry/examples/yao-meta-skill-1.0.0.json create mode 100644 registry/index.json create mode 100644 registry/index.schema.json create mode 100644 registry/package.schema.json create mode 100644 registry/packages/yao-meta-skill.json create mode 100644 reports/adoption_drift_report.json create mode 100644 reports/adoption_drift_report.md create mode 100644 reports/compiled_targets.json create mode 100644 reports/compiled_targets.md create mode 100644 reports/install_simulation.json create mode 100644 reports/install_simulation.md create mode 100644 reports/output_blind_answer_key.json create mode 100644 reports/output_blind_review_pack.json create mode 100644 reports/output_blind_review_pack.md create mode 100644 reports/output_execution_runs.json create mode 100644 reports/output_execution_runs.md create mode 100644 reports/output_review_adjudication.json create mode 100644 reports/output_review_adjudication.md create mode 100644 reports/package_verification.json create mode 100644 reports/package_verification.md create mode 100644 reports/registry_audit.json create mode 100644 reports/registry_audit.md create mode 100644 reports/review_annotations.json create mode 100644 reports/review_annotations.md create mode 100644 reports/review_waivers.json create mode 100644 reports/review_waivers.md create mode 100644 reports/runtime_permission_probes.json create mode 100644 reports/runtime_permission_probes.md create mode 100644 reports/upgrade_check.json create mode 100644 reports/upgrade_check.md create mode 100644 scripts/adjudicate_output_review.py create mode 100644 scripts/compile_skill.py create mode 100644 scripts/local_output_eval_runner.py create mode 100644 scripts/probe_runtime_permissions.py create mode 100644 scripts/registry_audit.py create mode 100644 scripts/render_adoption_drift_report.py create mode 100644 scripts/render_review_annotations.py create mode 100644 scripts/render_review_waivers.py create mode 100644 scripts/run_output_execution.py create mode 100644 scripts/simulate_install.py create mode 100644 scripts/upgrade_check.py create mode 100644 scripts/verify_package.py create mode 100644 security/network_policy.json create mode 100644 security/network_policy.md create mode 100644 security/permission_policy.json create mode 100644 security/permission_policy.md create mode 100644 skill_atlas/policy.json create mode 100644 tests/verify_adoption_drift.py create mode 100644 tests/verify_compile_skill.py create mode 100644 tests/verify_install_simulation.py create mode 100644 tests/verify_output_execution_runs.py create mode 100644 tests/verify_output_review_adjudication.py create mode 100644 tests/verify_package_verification.py create mode 100644 tests/verify_registry_audit.py create mode 100644 tests/verify_review_annotations.py create mode 100644 tests/verify_review_waivers.py create mode 100644 tests/verify_runtime_permission_probes.py create mode 100644 tests/verify_upgrade_check.py diff --git a/.gitignore b/.gitignore index 2c74bca..b4744fb 100644 --- a/.gitignore +++ b/.gitignore @@ -21,6 +21,7 @@ tests/tmp_system_model/ tests/tmp_security/ tests/tmp_baseline_compare.* reports/release_snapshots/ +reports/telemetry_events.jsonl .yao/ # Local business-skill experiments belong outside this meta-skill repo unless promoted intentionally. diff --git a/Makefile b/Makefile index 4095fab..659bca1 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -43,18 +43,42 @@ skill-report-charts-check: skill-ir-check: $(PYTHON) tests/verify_skill_ir.py +compiler-check: + $(PYTHON) tests/verify_compile_skill.py + output-eval-check: $(PYTHON) tests/verify_output_eval_lab.py +output-execution-check: + $(PYTHON) tests/verify_output_execution_runs.py + +output-review-adjudication-check: + $(PYTHON) tests/verify_output_review_adjudication.py + runtime-conformance-check: $(PYTHON) tests/verify_conformance_suite.py +runtime-permission-check: + $(PYTHON) tests/verify_runtime_permission_probes.py + trust-check: $(PYTHON) tests/verify_trust_check.py skill-atlas-check: $(PYTHON) tests/verify_skill_atlas.py +registry-audit-check: + $(PYTHON) tests/verify_registry_audit.py + +package-verify-check: + $(PYTHON) tests/verify_package_verification.py + +install-simulation-check: + $(PYTHON) tests/verify_install_simulation.py + +upgrade-check: + $(PYTHON) tests/verify_upgrade_check.py + review-viewer-check: $(PYTHON) tests/verify_review_viewer.py @@ -64,6 +88,15 @@ review-studio-check: feedback-check: $(PYTHON) tests/verify_feedback.py +adoption-drift-check: + $(PYTHON) tests/verify_adoption_drift.py + +review-waivers-check: + $(PYTHON) tests/verify_review_waivers.py + +review-annotations-check: + $(PYTHON) tests/verify_review_annotations.py + baseline-compare-check: $(PYTHON) tests/verify_baseline_compare.py @@ -154,11 +187,11 @@ sync-local-install: sync-active-install: $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check feedback-check adoption-drift-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_output_eval tests/tmp_conformance tests/tmp_trust tests/tmp_skill_atlas tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 2a44654..bda3dc8 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,20 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a front-loaded intent dialogue with an intent confidence gate, so the system keeps clarifying when the true job, outputs, exclusions, or standards are still fuzzy - a silent-by-default GitHub benchmark scan plus reference synthesis that studies top public repositories and world-class pattern tracks, then surfaces only real conflicts or uncertainty to the user - a generated visual HTML overview for each newly initialized skill -- a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, and release evidence +- a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, adoption drift, reviewer waivers, reviewer annotations, release evidence, and per-warning fix actions +- Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports +- a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks - a side-by-side HTML review studio for first-pass human review - an artifact design profile that defines visual direction, layout patterns, and quality gates for reports, tutorials, dashboards, screenshots, and review pages - a prompt quality profile that abstracts need modeling, RTF mapping, complexity, and quality checks into reviewer-visible evidence instead of bloating `SKILL.md` - a systems-thinking model that maps boundaries, feedback loops, drift risks, recurring failure patterns, and highest-leverage quality moves - three high-value next iteration directions after the first package is created - a lightweight feedback log that does not require a full promotion cycle +- a local-first metadata-only adoption and drift report that turns real usage signals into next iteration candidates - a baseline compare report for with-skill vs baseline review - a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits -- Skill IR as the platform-neutral semantic contract plus client-specific adapters +- Skill IR as the platform-neutral semantic contract, plus compiler reports and client-specific adapters +- Registry audit metadata with package version, owner, license, checksum, and compatibility matrix - governance, promotion, and portability checks built into the default flow ## Architecture @@ -85,7 +89,7 @@ Weighted score formula: `sum(score / 10 * weight)`. 2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit. 3. Let `quickstart` clarify intent first, then run silent benchmark scan and reference synthesis; it only surfaces explicit questions when intent is still unclear or when there is a real design conflict. 4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode. -5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/review-studio.html` to inspect release blockers and evidence paths in one page before adding more structure. +5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/review-studio.html` to inspect release blockers, permission approvals, and evidence paths in one page before adding more structure. Or use the unified authoring CLI: @@ -102,14 +106,26 @@ python3 scripts/yao.py artifact-design-profile my-skill python3 scripts/yao.py prompt-quality-profile my-skill python3 scripts/yao.py system-model my-skill python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before adding scripts." --rating 4 --category boundary +python3 scripts/yao.py adoption-drift my-skill --record-event skill_activation --activation-type explicit --outcome accepted +python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key trust-report --reviewer "Yao Team" --reason "Known warning accepted for this release with bounded follow-up." --expires-at 2026-09-30 +python3 scripts/yao.py review-waivers my-skill --add-waiver --gate-key permission-gates --reviewer "Yao Team" --reason "Permission warning accepted only for this non-governed release window." --expires-at 2026-09-30 +python3 scripts/yao.py review-annotations my-skill --add-annotation --gate-key output-lab --target-path reports/output_quality_scorecard.md --line 1 --body "Clarify recorded fixture vs model-executed evidence before release." python3 scripts/yao.py baseline-compare python3 scripts/yao.py check-update -python3 scripts/yao.py package . --platform generic --output-dir dist python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json +python3 scripts/yao.py compile-skill . --target openai --target claude --target generic +python3 scripts/yao.py package . --platform generic --output-dir dist python3 scripts/yao.py output-eval +python3 scripts/yao.py output-exec +python3 scripts/yao.py output-review python3 scripts/yao.py conformance . python3 scripts/yao.py trust . +python3 scripts/yao.py runtime-permissions . --package-dir dist python3 scripts/yao.py skill-atlas --workspace-root . +python3 scripts/yao.py registry-audit . +python3 scripts/yao.py package-verify . --package-dir dist --require-zip +python3 scripts/yao.py install-simulate . --package-dir dist +python3 scripts/yao.py upgrade-check . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json ``` ## Local Development Source @@ -160,7 +176,9 @@ python3 scripts/judge_blind_eval.py --description-file SKILL.md --cases evals/bl python3 scripts/context_sizer.py . python3 scripts/resource_boundary_check.py . python3 scripts/governance_check.py . --require-manifest +python3 scripts/compile_skill.py . python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip +python3 scripts/probe_runtime_permissions.py . --package-dir dist python3 tests/verify_packager_failures.py ``` @@ -179,7 +197,11 @@ python3 scripts/yao.py workspace-flow --target root --label first-pass python3 scripts/yao.py review-viewer my-skill python3 scripts/yao.py review --target root python3 scripts/yao.py release-snapshot --target root --label release-candidate +python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json +python3 scripts/yao.py compile-skill . python3 scripts/yao.py package . --platform openai --platform claude --platform generic --output-dir dist --zip +python3 scripts/yao.py runtime-permissions . --package-dir dist +python3 scripts/yao.py package-verify . --package-dir dist --require-zip python3 scripts/yao.py test ``` @@ -225,7 +247,9 @@ The homepage panel below is generated from the current eval suite so the family- Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md) -- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks and carry IR provenance plus semantic parity metadata +- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks and carry IR provenance, semantic parity metadata, and target-native behavior contracts +- target compiler validation: `openai`, `claude`, `generic`, and Agent Skills compatible contracts are compiled from Skill IR with generated-file mappings, adapter modes, native surfaces, permission enforcement notes, and unsupported-feature notes +- runtime permission probes: `openai`, `claude`, and `generic` adapters expose explicit permission contracts; current targets report `0` native-enforcement adapters and `3` metadata fallbacks with residual risks visible to reviewers - portability score: `100/100` with neutral activation, execution, trust, and degradation metadata preserved across all exported targets - description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately - judge-backed blind eval: root, team frontend review, and governed incident command now pass an independent rubric judge on blind holdout prompts @@ -234,8 +258,8 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - governance and resource-boundary checks are part of the default test path - root governance maturity score: `90/100`; governed benchmark example: `95/100` - CJK-aware trigger matching is now covered by explicit Chinese build, packaging, eval, and near-neighbor cases -- context budgets: root `994/1000`, complex benchmark `790/1000`, governed benchmark `760/1000` -- quality density: root `130.8`, complex benchmark `164.6`, governed benchmark `171.1` +- context budgets: root `987/1000`, complex benchmark `790/1000`, governed benchmark `760/1000` +- quality density: root `131.7`, complex benchmark `164.6`, governed benchmark `171.1` - regression milestones are tracked in [reports/regression_history.md](reports/regression_history.md) - description drift history is tracked in [reports/description_drift_history.md](reports/description_drift_history.md) - route confusion is tracked in [reports/route_scorecard.md](reports/route_scorecard.md) @@ -244,6 +268,14 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - candidate lifecycle states are published in [reports/candidate_registry.md](reports/candidate_registry.md) - lightweight with-skill vs baseline comparison is published in [reports/baseline-compare.md](reports/baseline-compare.md) - Review Studio 2.0 gate evidence is published in [reports/review-studio.html](reports/review-studio.html) +- Review Studio fix actions are embedded in [reports/review-studio.json](reports/review-studio.json) +- reviewer waiver evidence is published in [reports/review_waivers.md](reports/review_waivers.md) +- target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md) +- registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md) +- package archive verification is published in [reports/package_verification.md](reports/package_verification.md) +- temporary local install simulation is published in [reports/install_simulation.md](reports/install_simulation.md) +- upgrade diff, version-bump recommendation, and release-note evidence are published in [reports/upgrade_check.md](reports/upgrade_check.md) +- local-first adoption and drift telemetry is summarized in [reports/adoption_drift_report.md](reports/adoption_drift_report.md) - context budget summaries are tracked in [reports/context_budget.md](reports/context_budget.md) - portability status is tracked in [reports/portability_score.md](reports/portability_score.md) @@ -353,7 +385,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, package, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -366,10 +398,21 @@ Utility scripts that make the meta-skill operational: - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage - `render_skill_overview.py`: generates the white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, top-right language switch, v2 scorecard, inline SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap - `export_skill_ir.py`: exports the 2.0 platform-neutral Skill IR contract from `SKILL.md`, manifest, interface metadata, evals, resources, and reports -- `run_output_eval.py`: runs the Output Eval Lab v0 with static with-skill vs baseline assertion grading and writes `reports/output_quality_scorecard.*` +- `compile_skill.py`: compiles Skill IR into target-specific semantic contracts, generated-file maps, adapter modes, target-native behavior contracts, preserved semantics, warnings, and unsupported-feature notes +- `run_output_eval.py`: runs the Output Eval Lab v0 with static with-skill vs baseline assertion grading, blind A/B review pack generation, and separate answer key artifacts +- `run_output_execution.py`: records output-eval execution evidence, distinguishing recorded fixtures, command runners, and provider-backed model runs with timing and token metadata +- `local_output_eval_runner.py`: deterministic local runner for command-executed output-eval smoke evidence without claiming provider-backed model generation +- `adjudicate_output_review.py`: records reviewer choices for blind A/B output evals, compares them with the answer key, and renders pending, match, disagreement, and invalid-decision audit reports +- `render_review_annotations.py`: records reviewer annotations tied to Review Studio gates, source/report paths, and optional line numbers, with open blocker annotations reflected in Review Studio decisions - `run_conformance_suite.py`: verifies runtime conformance for OpenAI, Claude, Agent Skills, VS Code/Copilot-style, and generic targets -- `trust_check.py`: generates the trust/security report for scripts, dependencies, secret risk, trust metadata, and package integrity +- `trust_check.py`: generates the trust/security report for scripts, dependencies, secret risk, bounded network host policy, execution-level `--help` smoke checks, permission inputs, trust metadata, and stable source-contract integrity - `build_skill_atlas.py`: builds the Skill Atlas catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview for a multi-skill workspace +- `registry_audit.py`: builds registry package metadata and audits version, owner, license, checksum, Skill IR source, and compatibility matrix +- `verify_package.py`: verifies generated package manifests, target adapters, zip archive safety, archive checksum, and registry parity +- `simulate_install.py`: extracts a generated zip into a temporary skill root and verifies entrypoint, manifest, interface, reports, and adapters can be loaded +- `upgrade_check.py`: compares current and previous registry package metadata, recommends a version bump, and blocks incompatible upgrade claims +- `render_adoption_drift_report.py`: records metadata-only local telemetry and renders adoption, missed-trigger, bad-output, script-error, and review-drift signals without packaging raw event logs +- `render_review_waivers.py`: validates human reviewer risk approvals with gate keys, reasons, expiry dates, and blocker-safe waiver policy - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain - `check_update.py`: checks GitHub for a newer `VERSION` or remote manifest version and reports a reinstall hint without modifying local files - `render_output_risk_profile.py`: predicts output-specific failure modes such as generic headings, citation clutter, screenshot mistakes, weak Markdown tables, and missing execution assumptions @@ -444,6 +487,8 @@ The typical flow is: Examples: ```bash +python3 scripts/export_skill_ir.py ./yao-meta-skill --output-json ./yao-meta-skill/reports/skill-ir.json +python3 scripts/compile_skill.py ./yao-meta-skill --target openai --target claude --target generic python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --expectations evals/packaging_expectations.json --zip python3 scripts/context_sizer.py ./yao-meta-skill python3 scripts/resource_boundary_check.py ./yao-meta-skill diff --git a/SKILL.md b/SKILL.md index ea3762c..34d4901 100644 --- a/SKILL.md +++ b/SKILL.md @@ -32,11 +32,11 @@ Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes]( 6. Add only folders and gates that earn their keep. 7. Surface the top three next iteration directions. -Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Review Studio](references/review-studio-method.md). +Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Registry](references/distribution-registry-method.md), [Telemetry](references/telemetry-drift-method.md), [Waivers](references/review-waiver-method.md), [Review Studio](references/review-studio-method.md). ## Skill OS 2.0 Gates -For production, library, governed, or team-distributed work, use Skill IR, trigger + output eval, Skill Atlas, runtime conformance, trust report, and Review Studio before release. +For production, library, governed, or team-distributed work, use Skill IR, target compiler contracts, trigger + output eval, Skill Atlas, runtime conformance, trust report, registry metadata, package verification, install simulation, upgrade check, adoption/drift telemetry, reviewer waiver evidence, and Review Studio before release. ## First-Turn Style diff --git a/evals/README.md b/evals/README.md index 56d60e7..e1e848e 100644 --- a/evals/README.md +++ b/evals/README.md @@ -15,8 +15,15 @@ Contents: - `improved_description.txt`: current stronger trigger description - `../reports/description_optimization*.{json,md}`: generated route-optimization reports for the root skill - `failure-cases.md`: current weak spots and regression targets -- `packaging_expectations.json`: required packaging behaviors for supported targets, including IR provenance and semantic parity fields +- `packaging_expectations.json`: required packaging behaviors for supported targets, including IR provenance, compiler contracts, and semantic parity fields - `output/`: Output Eval Lab cases, fixtures, and schema for with-skill vs baseline assertion grading +- registry audit tests verify package metadata, compatibility, checksum, owner, license, and Skill IR provenance +- package verification tests verify generated manifests, target adapters, zip archive safety, archive checksum, and registry parity +- install simulation tests verify archive extraction, entrypoint loading, manifest/interface readability, adapter loading, and unsafe zip blocking +- upgrade check tests verify version-bump recommendations, target diff detection, compatibility regression blocking, and release-note evidence +- adoption drift tests verify metadata-only telemetry aggregation, privacy-field blocking, and exclusion of raw local telemetry logs from zip packages +- review waiver tests verify reviewer, reason, expiry, active coverage, expired records, invalid records, and Review Studio integration +- compiler tests verify Skill IR target contracts, adapter modes, generated-file mappings, unsupported targets, and markdown/json report artifacts - `../reports/`: generated suite JSON plus the homepage-visible family summary panel source Use: @@ -34,10 +41,24 @@ python3 scripts/render_iteration_ledger.py python3 tests/verify_description_optimization.py python3 tests/verify_route_confusion.py python3 tests/verify_failure_regressions.py +python3 scripts/compile_skill.py . --target openai --target claude --target generic +python3 tests/verify_compile_skill.py python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip +python3 scripts/verify_package.py . --package-dir dist --require-zip python3 tests/verify_packager_failures.py +python3 tests/verify_package_verification.py +python3 scripts/simulate_install.py . --package-dir dist +python3 tests/verify_install_simulation.py python3 scripts/run_output_eval.py python3 tests/verify_output_eval_lab.py +python3 scripts/registry_audit.py . +python3 tests/verify_registry_audit.py +python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json +python3 tests/verify_upgrade_check.py +python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted +python3 tests/verify_adoption_drift.py +python3 scripts/render_review_waivers.py . +python3 tests/verify_review_waivers.py ``` Regression scope now includes: @@ -58,3 +79,8 @@ Regression scope now includes: - calibration summaries that surface score gaps, threshold margins, and risk bands for each acceptance gate - family-level drift history that records which blind and adversarial families stay clean over time - output-quality scorecards that compare without-skill and with-skill artifacts through assertion grading, including near-neighbor and file-backed governed cases +- install-simulation checks that prove generated archives can be extracted into a temporary local skill root and loaded without touching the active global skills directory +- upgrade-readiness checks that compare registry package baselines, recommend semver bumps, and block breaking changes without an adequate declared version +- local-first adoption drift checks that turn accepted, edited, missed, rejected, script-error, and review-overdue metadata into next iteration candidates without storing raw prompts or outputs +- reviewer-waiver checks that keep warning acceptance explicit, expiring, and separate from non-waivable blockers +- compiler checks that turn Skill IR into reviewable target contracts before packaging embeds those contracts into adapters diff --git a/evals/output/schema.json b/evals/output/schema.json index 675e24a..f08d8b0 100644 --- a/evals/output/schema.json +++ b/evals/output/schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://yao.example/output-eval-case.schema.json", "title": "Yao Output Eval Case", - "description": "Static v0 output-eval case for comparing without-skill and with-skill outputs.", + "description": "Output-eval case for comparing without-skill and with-skill outputs, with optional execution metadata.", "type": "object", "additionalProperties": false, "required": ["id", "prompt", "baseline_output", "with_skill_output", "assertions"], @@ -57,6 +57,26 @@ "type": "object", "additionalProperties": true }, + "execution": { + "type": "object", + "description": "Optional case-level execution metadata for fixture, command, or provider-backed output runs.", + "additionalProperties": true, + "properties": { + "mode": { + "type": "string", + "enum": ["recorded_fixture", "command", "model"] + }, + "provider": { + "type": "string" + }, + "model": { + "type": "string" + }, + "holdout": { + "type": "boolean" + } + } + }, "metadata": { "type": "object", "additionalProperties": true diff --git a/evals/packaging_expectations.json b/evals/packaging_expectations.json index f7766b0..374c905 100644 --- a/evals/packaging_expectations.json +++ b/evals/packaging_expectations.json @@ -12,6 +12,12 @@ "ir_schema_version", "semantic_contract", "semantic_parity", + "compiler", + "compiled_contract", + "permission_contract", + "target_permission_contract", + "target_native_contract", + "target_transform", "canonical_metadata", "canonical_format", "activation_mode", diff --git a/references/distribution-registry-method.md b/references/distribution-registry-method.md new file mode 100644 index 0000000..68bb052 --- /dev/null +++ b/references/distribution-registry-method.md @@ -0,0 +1,57 @@ +# Distribution Registry Method + +Registry metadata turns a local skill directory into an installable and reviewable package record. + +## When To Use + +Use registry audit for library, governed, team-distributed, or upgrade-sensitive skills. Scaffold skills can skip it until reuse is real. + +## Required Evidence + +- package name, version, owner, maturity, review cadence, and license +- Skill IR source and schema version +- trust level and package SHA256 +- package verification status and archive SHA256 when a zip distribution is built +- install simulation status when an archive is meant to be installed locally or by a team +- adoption/drift aggregate status when local telemetry exists +- target compatibility matrix +- upgrade diff, recommended version bump, and breaking-change notes when a previous package baseline exists +- links to overview, Review Studio, trust report, conformance matrix, package verification, install simulation, adoption drift evidence, and review waivers + +## Release Rule + +Do not publish a team package when registry audit reports missing version, hash, owner, review cadence, license, valid Skill IR, or passing compatibility for declared targets. + +For an installable archive, first build the distribution and run package verification: + +```bash +python3 scripts/yao.py package . --platform openai --platform claude --platform generic --output-dir dist --zip +python3 scripts/yao.py package-verify . --package-dir dist --require-zip +python3 scripts/yao.py install-simulate . --package-dir dist +python3 scripts/yao.py registry-audit . +python3 scripts/yao.py upgrade-check . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json +``` + +Do not claim archive readiness when package verification reports unsafe zip paths, missing target adapters, missing package manifest, registry metadata mismatch, or unreadable archive contents. + +Do not claim install readiness when install simulation cannot extract the archive into a temporary skill root, load `SKILL.md` frontmatter, read `manifest.json`, read `agents/interface.yaml`, find the overview and Review Studio reports, or load each generated adapter. + +Do not include raw `reports/telemetry_events.jsonl` in a distributed package. Include only aggregate adoption drift reports, and block release review when telemetry contains raw prompts, outputs, transcripts, notes, or messages. + +Review waiver evidence may be distributed as `reports/review_waivers.md/json` because it is metadata-only reviewer accountability. Do not store raw prompts, outputs, transcripts, credentials, or private customer detail in waiver reasons. + +Do not claim upgrade readiness when upgrade check reports an insufficient version bump, target removal without a major bump, compatibility regression without a major bump, package name change, or invalid semver. Include `reports/upgrade_check.md` in the reviewer evidence bundle so release notes and migration guidance are tied to the exact registry diff. + +## Reviewer Gate + +A reviewer should be able to answer: + +1. Which package version is being installed? +2. Who owns the package? +3. Which targets are compatible? +4. Which checksum identifies the reviewed package contents? +5. Which reports prove trust and runtime readiness? +6. Was the installable archive verified, and which checksum identifies it? +7. Was the archive install-simulated in a temporary local skill root? +8. What changed since the previous package, and does the declared version bump match the recommended bump? +9. Are adoption and drift signals summarized without packaging raw local telemetry? diff --git a/references/output-eval-method.md b/references/output-eval-method.md index 02947a9..d9d999c 100644 --- a/references/output-eval-method.md +++ b/references/output-eval-method.md @@ -38,10 +38,63 @@ The first v0 scorecard reports: - with-skill pass rate - absolute delta - failed assertions and failure taxonomy +- execution mode, timing, and token evidence when `reports/output_execution_runs.md` is generated +- blind A/B review pack count - recommended next fixes Production promotion should require the with-skill pass rate to beat baseline and should explain every failed assertion. +## Execution Evidence + +Run execution evidence after the scorecard: + +```bash +python3 scripts/yao.py output-exec +``` + +By default, this records the current case outputs as `recorded_fixture`. That is useful for reproducibility, but it is not model-executed evidence. To collect real run evidence, pass `--runner-command` with a command or JSON string list. The runner receives a JSON request on stdin and should return JSON with: + +- `output` +- optional `execution_kind`: `command` or `model` +- optional `provider` and `model` +- optional `usage.input_tokens`, `usage.output_tokens`, and `usage.total_tokens` + +Only runs that return provider/model metadata or `execution_kind: "model"` should count as model-executed. If token usage is absent, the report may estimate tokens, but the estimate must be labeled as estimated. + +For local release-gate smoke evidence without external model credentials, use the deterministic runner: + +```bash +python3 scripts/yao.py output-exec --runner-command '["python3","scripts/local_output_eval_runner.py"]' +``` + +This verifies the command-runner contract, timing capture, grading path, and failure handling. It must not be described as provider-backed model evidence. + +## Blind A/B Review + +Every output eval run should also generate: + +- `reports/output_blind_review_pack.md` +- `reports/output_blind_review_pack.json` +- `reports/output_blind_answer_key.json` + +The review pack must hide whether Variant A or Variant B came from the baseline or the skill-guided output. The answer key is separate audit evidence and should only be opened after a reviewer has made a judgment. + +## Reviewer Adjudication + +After blind review, record reviewer choices in `reports/output_review_decisions.json` and run: + +```bash +python3 scripts/adjudicate_output_review.py --write-template +python3 scripts/yao.py output-review +``` + +The adjudication report writes: + +- `reports/output_review_adjudication.json` +- `reports/output_review_adjudication.md` + +When no reviewer decisions exist, the report should say the cases are pending. Do not count pending cases as human agreement. Only a real `winner_variant` of `A` or `B` should contribute to agreement rate, disagreement count, and reviewer judgment count. + ## Anti-Overfitting Keep a small public smoke set and a separate holdout set. Rotate real failures into the taxonomy instead of editing only the prompt that failed. Add near-neighbor cases whenever the output looks good but the boundary is still unclear. diff --git a/references/packaging-contracts.md b/references/packaging-contracts.md index a760f78..b734d76 100644 --- a/references/packaging-contracts.md +++ b/references/packaging-contracts.md @@ -1,6 +1,6 @@ # Packaging Contracts -`cross_packager.py` is not just an export helper. It defines and validates platform contracts. +`cross_packager.py` is not just an export helper. It validates platform contracts and embeds target compiler output from `compile_skill.py`. ## Current Targets @@ -15,8 +15,13 @@ Each target contract defines: - required output fields - required output files - field mapping from the neutral source metadata +- compiled contract from Skill IR +- target transform metadata, including generated files and unsupported features - portable execution metadata - trust-boundary metadata +- permission contract metadata from the trust report +- target-specific permission representation and reviewer notes +- target-native behavior contract for native surface, activation policy, resource strategy, script strategy, permission enforcement, install scope, review artifacts, and fallback behavior - degradation strategy metadata ## Failure Handling @@ -27,6 +32,8 @@ When `--expectations` is provided: - missing required fields cause exit code `2` - validation failures are emitted in the JSON report +After packaging, run `scripts/probe_runtime_permissions.py` against the generated package directory. Packaging creates the permission metadata; the runtime permission probe verifies that each target adapter exposes the contract, target-specific representation, native-enforcement flag, operator note, and residual metadata-fallback risk. + ## Source Of Truth The platform-neutral semantic source is Skill IR when it exists: @@ -39,10 +46,13 @@ The structural validation sources remain: - `SKILL.md` - `agents/interface.yaml` -Target-specific metadata is generated at packaging time. The adapter must carry -`ir_source`, `ir_schema_version`, `job_to_be_done`, `semantic_contract`, and -`semantic_parity` so reviewers can see whether the target preserved the core -skill meaning or fell back to frontmatter-only metadata. +Target-specific metadata is generated through `scripts/compile_skill.py` and +then embedded at packaging time. The adapter must carry `compiler`, +`compiled_contract`, `permission_contract`, `target_permission_contract`, +`target_native_contract`, `target_transform`, `ir_source`, `ir_schema_version`, +`job_to_be_done`, `semantic_contract`, and `semantic_parity` so reviewers can +see whether the target preserved the core skill meaning or fell back to +frontmatter-only metadata. ## Portability Model @@ -51,7 +61,10 @@ The packaging layer now preserves four portable semantics from the neutral sourc - activation - execution - trust +- permissions - degradation - platform-neutral skill meaning from Skill IR +- target-specific native behavior notes for activation, resources, scripts, permission enforcement, install scope, review artifacts, and fallback behavior +- target-specific compile notes for generated files, adapter mode, preserved semantics, and unsupported features This means portability is not just "can it export a file?" but also "does the exported target preserve the source package's activation and safety assumptions?" diff --git a/references/platform-capability-matrix.md b/references/platform-capability-matrix.md index f06a259..101763d 100644 --- a/references/platform-capability-matrix.md +++ b/references/platform-capability-matrix.md @@ -2,17 +2,20 @@ This matrix describes the current packaging targets and their support level. -| Target | Metadata Adapter | Output Contract | Snapshot Test | Portability Semantics | Notes | -| --- | --- | --- | --- | --- | --- | -| `openai` | Yes | Yes | Yes | activation, execution, trust, degradation | Generates `targets/openai/agents/openai.yaml` | -| `claude` | Yes | Yes | Yes | activation, execution, trust, degradation | Generates `targets/claude/README.md` plus adapter metadata | -| `generic` | Yes | Yes | Yes | activation, execution, trust, degradation | Uses neutral adapter metadata only | +| Target | Metadata Adapter | Compiler Contract | Native Behavior Contract | Output Contract | Snapshot Test | Portability Semantics | Notes | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `openai` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Generates `targets/openai/agents/openai.yaml` | +| `claude` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Generates `targets/claude/README.md` plus adapter metadata | +| `generic` | Yes | Yes | Yes | Yes | Yes | activation, execution, trust, permissions, degradation, native behavior | Uses neutral adapter metadata only | +| `agent-skills-compatible` | Neutral source | Yes | Yes | Source-compatible | Yes | activation, execution, trust, permissions, degradation, native behavior | Keeps canonical `SKILL.md` plus `agents/interface.yaml` source shape | ## Current Support Model -- `openai`: strongest metadata adapter support -- `claude`: lightweight compatibility adapter, behavior still relies mainly on neutral source files -- `generic`: lowest-friction export for neutral Agent Skills consumers +- `openai`: strongest metadata adapter support with an explicit compiler contract. +- `claude`: lightweight compatibility adapter with an explicit compiler contract and fallback notes. +- `generic`: lowest-friction export for neutral Agent Skills consumers. +- `agent-skills-compatible`: canonical source shape with compiler evidence for review and distribution. +- runtime permission probes currently report metadata fallback for generated targets; no target is claimed as native-enforced until a client or installer integration can actually enforce the permission model. ## Portable Semantics @@ -21,15 +24,19 @@ Each target now preserves: - activation mode and optional path filters - execution context and shell choice - trust tier and remote inline-execution policy +- permission contract for network, file-write, subprocess, and interactive script surfaces +- target-native behavior contract for native surface, activation policy, resource strategy, script strategy, permission enforcement, install scope, review artifacts, and fallback behavior - degradation strategy for unsupported client behavior +- generated-file mapping and adapter mode from `reports/compiled_targets.json` ## Explicit Non-Goals This project does not yet implement: -- runtime-specific behavior transforms - client SDK integration - provider-specific execution logic +- provider-native installer actions or account-level activation changes +- native runtime permission enforcement ## Degradation Rule diff --git a/references/review-studio-method.md b/references/review-studio-method.md index 11b7614..da67a4c 100644 --- a/references/review-studio-method.md +++ b/references/review-studio-method.md @@ -6,19 +6,27 @@ Review Studio is the release-facing audit surface for a skill package. It does n - Show release blockers and warnings before the package deepens. - Link every gate back to a concrete evidence artifact. +- Generate review actions for every blocker and warning, with source-fix location and verification command. +- Make human warning acceptance auditable through a waiver ledger. +- Make reviewer comments auditable through an annotation ledger tied to gates, source/report paths, and optional line numbers. - Keep review flow vertical: summary first, gates second, supporting details after. -- Avoid hiding output quality, runtime, trust, and portfolio issues across separate pages. +- Avoid hiding output quality, runtime, trust, portfolio, and operating-loop issues across separate pages. ## Required Gates 1. Intent Canvas: intent confidence and unresolved input/output/exclusion gaps. 2. Trigger Lab: route scorecard, misroutes, ambiguous cases, and near-neighbor safety. -3. Output Lab: with-skill vs baseline delta, case count, file-backed cases, near-neighbor cases, and boundary cases. +3. Output Lab: with-skill vs baseline delta, execution mode, timing/token evidence, case count, file-backed cases, near-neighbor cases, boundary cases, blind A/B review pack evidence, and reviewer adjudication status. 4. Context Budget: initial load, budget tier, warnings, and quality density. 5. Runtime Matrix: target conformance pass/fail and degradation notes. 6. Trust Report: secret scan, script surface, dependency pinning, network/interactive flags, and package hash. -7. Skill Atlas: route collisions, stale skills, owner gaps, and no-route opportunities. -8. Release Notes: promotion status, migration notes, known gaps, and next move. +7. Permission Gates: reviewer-approved capability scope, reason, expiry, evidence, and target-enforcement notes. +8. Runtime Permission Probes: packaged adapter permission contracts, native-enforcement flags, metadata fallback notes, and residual risks. +9. Skill Atlas: route collisions, stale skills, owner gaps, and no-route opportunities. +10. Operations Loop: local-first metadata-only adoption, missed-trigger, bad-output, script-error, and review-drift signals. +11. Review Waivers: human risk approvals, active coverage, expired approvals, invalid records, and expiry policy. +12. Registry Audit: package metadata, install evidence, compatibility entries, and archive/source checksums. +13. Release Notes: promotion status, migration notes, known gaps, and next move. ## Gate Semantics @@ -27,3 +35,51 @@ Review Studio is the release-facing audit surface for a skill package. It does n - `block`: do not claim production, library, governed, or public readiness until fixed. For library and governed skills, Output Lab should have at least five cases and cover file-backed, near-neighbor, and boundary scenarios. + +Production, library, and governed reviews should also show a blind A/B review pack. The Review Studio gate may warn when scorecard evidence exists but no blind pack is present, because the package can prove assertions but not yet reduce reviewer bias. + +When `reports/output_execution_runs.json` exists, Review Studio should show the number of variant runs, command-executed runs, model-executed runs, recorded fixtures, timing-observed runs, and token-estimated runs. Recorded fixtures are valid reproducibility evidence, but they must not be described as model-executed output evidence. + +When `reports/output_review_adjudication.json` exists, Review Studio should show reviewed pairs and pending pairs. Pending reviewer decisions are acceptable as an explicit state, but they must not be counted as agreement or human review evidence. Invalid adjudication records should block release because they make the blind review audit untrustworthy. + +The Operations Loop must never display raw telemetry logs. It should link only to `reports/adoption_drift_report.md`; privacy or schema violations are blockers. + +The Runtime Permission Probes gate is evaluated after packaging, because it reads generated target adapters. A missing probe can warn in lighter modes, but governed release should not claim target permission readiness without `reports/runtime_permission_probes.json`. + +The Review Waivers gate must never convert a blocker into a pass. Waivers only cover warning-level risks, require reviewer, reason, scope, and expiry, and must link only to `reports/review_waivers.md`. + +Review Annotations are not waivers. They are reviewer comments attached to a Review Studio gate plus a relative source/report path and optional line number. Use them to preserve review context, requested edits, and source-line notes. Open blocker annotations should make the Review Studio decision `blocked` until the annotation is resolved or deferred with reviewer rationale. Open warning annotations can move the package into review, but they do not create gate-specific `review_actions`; actions remain reserved for non-pass gates. + +## Review Actions + +Every non-pass gate must produce a `review_actions` entry in `reports/review-studio.json`. When all gates pass, `review_actions` should be an empty list and the page should explicitly say there are no blocker or warning actions. + +Each action must include: + +- `gate_key` +- `status` +- `summary` +- `why` +- `source_fix` +- `source_refs` +- `evidence` +- `verification_command` + +`source_refs` must be structured entries with relative `path`, human label, kind, existence flag, best-effort line number, and relative link when the file exists. They should point to the smallest useful report or source file, not just a broad directory. + +The HTML page should render these actions before the detailed supporting sections so a reviewer can move directly from warning to fix. Action entries do not change gate count or score; they make the current decision more operational. + +## Review Annotations + +`reports/review_annotations.json` is the structured ledger, and `reports/review_annotations.md` is the human-readable review note surface. Each annotation should include: + +- `gate_key` +- `target_path` +- `line` when a useful source line exists +- `severity` +- `status` +- `reviewer` +- `body` +- optional `suggested_action` + +The ledger should reject absolute paths or paths that escape the skill directory. Missing target files are allowed as visible evidence gaps, not hidden failures. diff --git a/references/review-waiver-method.md b/references/review-waiver-method.md new file mode 100644 index 0000000..5872d83 --- /dev/null +++ b/references/review-waiver-method.md @@ -0,0 +1,57 @@ +# Review Waiver Method + +Review waivers make human risk acceptance explicit. They are not a way to hide problems; they are a local audit record for warning-level risks that the reviewer intentionally accepts for a bounded release window. + +## When To Use + +Use a waiver when: + +- Review Studio shows a warning that is understood and intentionally accepted. +- The warning cannot be fixed before release without a worse tradeoff. +- A reviewer can name the reason, scope, evidence, and expiry date. + +Do not use a waiver for blocker gates in v0. Blockers must be fixed before production, library, governed, or public readiness is claimed. In governed mode, missing or invalid high-permission approvals are blockers and should be fixed in `security/permission_policy.json`, not waived. + +## Required Fields + +Every waiver must include: + +- `gate_key`: the Review Studio gate being accepted. +- `decision`: `accepted-risk`, `false-positive`, or `temporary-exception`. +- `reviewer`: the accountable human or team. +- `reason`: a concrete reason of at least 20 characters. +- `created_at`: ISO date. +- `expires_at`: ISO date. +- `evidence`: optional path or note that explains the decision. +- `scope`: default `current-release`. + +## Release Semantics + +- Invalid waiver records block Review Studio. +- Expired waiver records stay visible and no longer cover warnings. +- Active waivers cover only the exact gate key they name. +- A warning without an active waiver remains visible as a warning. +- Raw user prompts, outputs, credentials, and private transcripts must not be stored in waiver reasons. + +## Commands + +Render or validate the ledger: + +```bash +python3 scripts/render_review_waivers.py . +``` + +Add a bounded approval: + +```bash +python3 scripts/yao.py review-waivers . \ + --add-waiver \ + --gate-key trust-report \ + --reviewer "Yao Team" \ + --reason "Network-capable scripts are documented and bounded for this release." \ + --expires-at 2026-09-30 +``` + +For a non-governed release where `permission-gates` is only a warning, the same command can name `--gate-key permission-gates`. Governed releases must instead provide reviewer, scope, reason, expiry, evidence, and target-enforcement fields in `security/permission_policy.json`. + +Review Studio reads `reports/review_waivers.json` and links to `reports/review_waivers.md`. diff --git a/references/skill-atlas-method.md b/references/skill-atlas-method.md index 4bcb473..a0f0455 100644 --- a/references/skill-atlas-method.md +++ b/references/skill-atlas-method.md @@ -17,6 +17,12 @@ Single-skill quality is not enough for a team library. A skill portfolio also ne - Flag stale skills based on `updated_at` and `review_cadence`. - Extract no-route opportunities from failure notes. +## Scope Policy + +Atlas keeps a full catalog, but release gates should distinguish actionable library skills from examples and test fixtures. + +Use `skill_atlas/policy.json` to mark path prefixes as non-actionable when they are intentionally retained as examples, evolution snapshots, embedded generated skills, or validator fixtures. Non-actionable items still appear in the full report, route matrix, stale list, and owner gap list, but Review Studio should use the actionable counts for release readiness. + ## Reviewer Gate -Use Atlas before promoting a single skill into a shared library. If a route collision or missing owner appears, fix the portfolio boundary before adding more local complexity to one skill. +Use Atlas before promoting a single skill into a shared library. If an actionable route collision, missing owner, or stale governed skill appears, fix the portfolio boundary before adding more local complexity to one skill. Non-actionable issues should stay visible as evidence, not as release blockers. diff --git a/references/telemetry-drift-method.md b/references/telemetry-drift-method.md new file mode 100644 index 0000000..8633108 --- /dev/null +++ b/references/telemetry-drift-method.md @@ -0,0 +1,60 @@ +# Telemetry And Drift Method + +Telemetry turns real use into the next iteration queue. It must stay local-first and metadata-only by default. + +## When To Use + +Use the telemetry drift loop when a skill is production, library, governed, team-distributed, or repeatedly invoked by more than one workflow. + +Do not collect raw prompts, model outputs, transcripts, notes, messages, or private files. If a reviewer needs examples, store anonymized fixtures separately and cite them as eval evidence, not telemetry. + +## Event Contract + +The local event stream is `reports/telemetry_events.jsonl`. It is intentionally narrow: + +```json +{ + "event": "skill_activation", + "skill": "example-skill", + "version": "2.0.0", + "activation_type": "implicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-13T10:00:00Z" +} +``` + +Allowed events: `skill_activation`, `skill_output`, `script_run`, `review_event`. + +Allowed outcomes: `accepted`, `edited`, `rejected`, `missed`, `failed`, `reviewed`, `unknown`. + +Allowed failure types: `wrong_trigger`, `under_trigger`, `bad_output`, `missing_resource`, `script_error`, `review_overdue`, `none`. + +## Privacy Rule + +The raw JSONL event log is local evidence and should not be distributed in skill packages. The distributable artifact is the aggregate report: + +- `reports/adoption_drift_report.json` +- `reports/adoption_drift_report.md` + +Package builders should exclude `reports/telemetry_events.jsonl`. The root repository also ignores this raw event stream so local usage evidence does not become ordinary source history by accident. + +## Release Interpretation + +- `no-data`: acceptable for a first scaffold, but a warning for governed release review. +- `low`: events exist and no drift failure signal is present. +- `medium`: at least one missed trigger, wrong trigger, bad output, script error, or overdue review signal exists. +- `high`: several drift signals are present; convert them into eval cases or governance actions before calling the skill release-ready. + +## Iteration Loop + +1. Capture metadata-only events locally. +2. Render `reports/adoption_drift_report.md`. +3. Convert missed triggers into trigger eval cases. +4. Convert bad outputs into Output Eval assertions and failure taxonomy entries. +5. Convert script errors into non-interactive smoke tests. +6. Feed review-overdue signals back into Skill Atlas and owner review. + +## Review Studio Role + +Review Studio should show the aggregate telemetry gate as an operating loop, not as raw logs. A blocker means the telemetry contract was violated. A warning means the evidence is absent or the drift signal needs a follow-up case. diff --git a/references/trust-security-method.md b/references/trust-security-method.md index 444820d..ad6c9f5 100644 --- a/references/trust-security-method.md +++ b/references/trust-security-method.md @@ -16,11 +16,63 @@ Run the trust report when: - obvious secret patterns - script help surface and interactive prompts +- execution-level `--help` smoke checks - dependency pinning - runtime trust metadata - network-capable scripts -- package integrity digest +- bounded host policy for network-capable scripts +- reviewer-approved permission policy for high-permission capabilities +- packaged-target runtime permission probes for adapter contracts and metadata fallback limits +- source-contract integrity digest + +## Script Interface Rule + +Every Python file under `scripts/` is reviewed as part of the package trust surface. + +- CLI scripts should use `argparse` so reviewers and installers can run `python3 scripts/name.py --help` before execution. +- The trust report executes `python3 scripts/name.py --help` for CLI scripts with `argparse`, with a short timeout, and records pass/fail evidence. +- Import-only modules should declare `SCRIPT_INTERFACE = "internal-module"` near the top of the file. +- Internal modules should also declare `SCRIPT_INTERFACE_REASON` with a short explanation of which CLI or renderer imports them. +- The trust report keeps internal modules in the script inventory, but excludes them from CLI help-surface warnings. +- A Python file without an explicit internal-module declaration is treated as a CLI script by default. +- CLI scripts without `argparse` are not smoke-executed; they remain visible as help-surface warnings. + +## Network Policy Rule + +Network-capable scripts must be bounded by a machine-readable policy before team distribution. + +- Put the policy in `security/network_policy.json`. +- Add one entry per network-capable script under `scripts`. +- Declare `allowed_hosts`, `allowed_path_prefixes`, purpose, timeout, auth mode, and custom-host behavior. +- Default to HTTPS-only and deny custom hosts unless a CLI flag or environment variable makes the override explicit. +- The trust report compares HTTPS URL literals in each script with `allowed_hosts`; missing or mismatched entries remain reviewer-visible warnings. + +## Permission Approval Rule + +High-permission capabilities must be approved before governed release. + +- Put approvals in `security/permission_policy.json`. +- Cover each required capability detected by the trust report: `network`, `file_write`, `subprocess`, and `interactive` when present. +- Each approval must include `decision: approved`, `reviewer`, `scope`, `reason`, `expires_at`, `evidence`, and `target_enforcement`. +- Review Studio surfaces these checks as the `permission-gates` gate. +- Missing, invalid, or expired approvals block governed mode. They remain visible warnings in lighter modes. + +## Runtime Permission Probe Rule + +Permission approval validates reviewer intent. Runtime permission probes validate the generated target adapters after packaging. + +- Run `python3 scripts/probe_runtime_permissions.py . --package-dir dist` after `cross_packager.py`. +- The probe writes `reports/runtime_permission_probes.json` and `reports/runtime_permission_probes.md`. +- A passing probe requires every target adapter to carry `permission_contract`, `target_permission_contract`, declared capabilities, a native-enforcement boolean, representation notes, and operator notes. +- If a target has no native enforcement, the probe must mark an explicit metadata fallback and keep residual risk reviewer-visible. +- Review Studio surfaces this as the `permission-runtime` gate. ## Release Rule High-risk secrets or unrestricted remote inline execution block governed release. Warnings are reviewer-visible but do not block v0 unless the release owner decides the target environment requires stricter policy. + +## Hash Scope + +`package_sha256` is a stable source-contract digest, not a generated archive digest. It covers the skill entrypoint, metadata, scripts, references, evals, runtime, templates, security notes, Skill IR, and root control files. It deliberately excludes generated `reports/`, packaged `dist/` archives, and raw local telemetry so a report render or local adoption log cannot mutate the trust fingerprint. + +Use the package verification or registry audit report for the distributable archive checksum. diff --git a/registry/examples/yao-meta-skill-1.0.0.json b/registry/examples/yao-meta-skill-1.0.0.json new file mode 100644 index 0000000..2d5cd36 --- /dev/null +++ b/registry/examples/yao-meta-skill-1.0.0.json @@ -0,0 +1,43 @@ +{ + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.0.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", + "targets": [ + "openai", + "claude", + "generic" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "0000000000000000000000000000000000000000000000000000000000000000" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "generic": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": false, + "archive_sha256": "", + "package_verification": "" + }, + "generated_at": "2026-06-12" +} diff --git a/registry/index.json b/registry/index.json new file mode 100644 index 0000000..46e108d --- /dev/null +++ b/registry/index.json @@ -0,0 +1,21 @@ +{ + "schema_version": "2.0", + "generated_at": "2026-06-13", + "package_count": 1, + "packages": [ + { + "name": "yao-meta-skill", + "version": "1.1.0", + "maturity": "governed", + "owner": "Yao Team", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "package_metadata": "registry/packages/yao-meta-skill.json", + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" + } + ] +} diff --git a/registry/index.schema.json b/registry/index.schema.json new file mode 100644 index 0000000..b48881e --- /dev/null +++ b/registry/index.schema.json @@ -0,0 +1,30 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.example/registry-index.schema.json", + "title": "Yao Skill Registry Index", + "type": "object", + "additionalProperties": false, + "required": ["schema_version", "generated_at", "package_count", "packages"], + "properties": { + "schema_version": {"const": "2.0"}, + "generated_at": {"type": "string"}, + "package_count": {"type": "integer", "minimum": 0}, + "packages": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "version", "maturity", "owner", "targets", "package_metadata", "package_sha256"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "maturity": {"type": "string", "minLength": 1}, + "owner": {"type": "string", "minLength": 1}, + "targets": {"type": "array", "items": {"type": "string"}}, + "package_metadata": {"type": "string", "minLength": 1}, + "package_sha256": {"type": "string", "minLength": 64, "maxLength": 64} + } + } + } + } +} diff --git a/registry/package.schema.json b/registry/package.schema.json new file mode 100644 index 0000000..f714b79 --- /dev/null +++ b/registry/package.schema.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.example/registry-package.schema.json", + "title": "Yao Skill Registry Package", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "name", + "version", + "description", + "targets", + "maturity", + "owner", + "review_cadence", + "trust_level", + "license", + "checksums", + "compatibility", + "source", + "artifacts", + "generated_at" + ], + "properties": { + "schema_version": {"const": "2.0"}, + "name": {"type": "string", "minLength": 1}, + "version": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1}, + "targets": {"type": "array", "items": {"type": "string"}}, + "maturity": {"type": "string", "minLength": 1}, + "owner": {"type": "string", "minLength": 1}, + "review_cadence": {"type": "string", "minLength": 1}, + "trust_level": {"type": "string", "minLength": 1}, + "license": {"type": "string", "minLength": 1}, + "checksums": { + "type": "object", + "additionalProperties": false, + "required": ["package_sha256"], + "properties": { + "package_sha256": {"type": "string", "minLength": 64, "maxLength": 64}, + "archive_sha256": {"type": "string", "minLength": 64, "maxLength": 64} + } + }, + "compatibility": { + "type": "object", + "additionalProperties": {"enum": ["pass", "warn", "fail", "missing", "unknown"]} + }, + "source": {"type": "object"}, + "artifacts": {"type": "object"}, + "distribution": {"type": "object"}, + "generated_at": {"type": "string"} + } +} diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json new file mode 100644 index 0000000..72a338c --- /dev/null +++ b/registry/packages/yao-meta-skill.json @@ -0,0 +1,56 @@ +{ + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe", + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "agent-skills": "pass", + "vscode": "pass", + "generic": "pass", + "agent-skills-compatible": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "package_verification": "reports/package_verification.json", + "install_simulated": true, + "install_simulation": "reports/install_simulation.json" + }, + "generated_at": "2026-06-13" +} diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json new file mode 100644 index 0000000..8575418 --- /dev/null +++ b/reports/adoption_drift_report.json @@ -0,0 +1,76 @@ +{ + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13T09:57:51Z", + "skill_dir": ".", + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "summary": { + "event_count": 1, + "activation_count": 1, + "accepted_count": 1, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 100.0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "low", + "event_types": { + "skill_activation": 1 + }, + "failure_types": {} + }, + "adoption_by_skill": [ + { + "skill": "yao-meta-skill", + "events": 1, + "accepted": 1, + "edited": 0, + "rejected": 0, + "missed": 0, + "adoption_rate": 100.0 + } + ], + "next_iteration_candidates": [], + "recent_events": [ + { + "event": "skill_activation", + "skill": "yao-meta-skill", + "version": "1.1.0", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-13T10:00:00Z" + } + ], + "failures": [], + "artifacts": { + "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "json": "reports/adoption_drift_report.json", + "markdown": "reports/adoption_drift_report.md" + } +} diff --git a/reports/adoption_drift_report.md b/reports/adoption_drift_report.md new file mode 100644 index 0000000..743dd61 --- /dev/null +++ b/reports/adoption_drift_report.md @@ -0,0 +1,35 @@ +# Adoption And Drift Report + +Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, transcripts, and notes are not allowed in the event stream. + +## Summary + +- Events: `1` +- Activation events: `1` +- Adoption rate: `100.0` +- Missed trigger signals: `0` +- Bad output signals: `0` +- Script error signals: `0` +- Review overdue signals: `0` +- Risk band: `low` + +## Privacy Contract + +- Storage is local-first. +- Events are metadata-only. +- Raw user prompts, model outputs, transcripts, notes, and messages are blocked. +- Distributed packages should include this aggregate report, not raw `reports/telemetry_events.jsonl`. + +## Adoption By Skill + +| Skill | Events | Accepted | Edited | Rejected | Missed | Adoption Rate | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| `yao-meta-skill` | 1 | 1 | 0 | 0 | 0 | 100.0 | + +## Next Iteration Candidates + +- No telemetry-driven iteration candidate yet. + +## Recent Metadata Events + +- `2026-06-13T10:00:00Z` `yao-meta-skill` event=`skill_activation` activation=`explicit` outcome=`accepted` failure=`none` diff --git a/reports/artifact-design-profile.json b/reports/artifact-design-profile.json index 7f52095..2e76f3c 100644 --- a/reports/artifact-design-profile.json +++ b/reports/artifact-design-profile.json @@ -1,6 +1,6 @@ { "skill_name": "yao-meta-skill", - "design_system": "review studio", + "design_system": "metric editorial", "primary_artifact": { "key": "code_or_cli", "label": "Code, CLI, or implementation guide", @@ -36,12 +36,22 @@ { "key": "review_viewer", "label": "Review viewer", - "score": 1, + "score": 2, "matched_keywords": [ - "review" + "review", + "viewer" ], "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." }, + { + "key": "dashboard", + "label": "Dashboard or metrics page", + "score": 1, + "matched_keywords": [ + "table" + ], + "direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation." + }, { "key": "visual_capture", "label": "Screenshot or visual evidence", diff --git a/reports/artifact-design-profile.md b/reports/artifact-design-profile.md index 000e94b..4b70889 100644 --- a/reports/artifact-design-profile.md +++ b/reports/artifact-design-profile.md @@ -1,7 +1,7 @@ # Artifact Design Profile Skill: `yao-meta-skill` -Design system: `review studio` +Design system: `metric editorial` ## Primary Artifact Direction @@ -22,10 +22,15 @@ Execution-focused technical artifact with environment assumptions, copyable comm - Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail. ### Review viewer -- Matched keywords: review -- Score: `1` +- Matched keywords: review, viewer +- Score: `2` - Direction: Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix. +### Dashboard or metrics page +- Matched keywords: table +- Score: `1` +- Direction: Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation. + ### Screenshot or visual evidence - Matched keywords: screenshot - Score: `1` diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json new file mode 100644 index 0000000..0f32d4d --- /dev/null +++ b/reports/compiled_targets.json @@ -0,0 +1,3130 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-13", + "skill_dir": ".", + "summary": { + "target_count": 4, + "pass_count": 4, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "schema_version": "1.0", + "target": "openai", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "openai", + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "degradation_strategy": "metadata-adapter", + "target_runtime": { + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "target_transform": { + "target": "openai", + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "metadata-adapter", + "permission_representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "permission_enforcement": "metadata-only" + }, + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "claude", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "claude", + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "degradation_strategy": "neutral-source-plus-adapter", + "target_runtime": { + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Claude-compatible neutral source folder with adapter notes" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "target_transform": { + "target": "claude", + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source-plus-adapter", + "permission_representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "permission_enforcement": "metadata-fallback" + }, + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "generic", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "generic", + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills compatible neutral package" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "target_transform": { + "target": "generic", + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "targets/generic/adapter.json::target_permission_contract", + "native_surface": "Agent Skills compatible neutral package", + "activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "agent-skills-compatible", + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills standard source tree" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "target_transform": { + "target": "agent-skills-compatible", + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "adapter metadata", + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "json": "reports/compiled_targets.json", + "markdown": "reports/compiled_targets.md" + } +} diff --git a/reports/compiled_targets.md b/reports/compiled_targets.md new file mode 100644 index 0000000..ea2ddcb --- /dev/null +++ b/reports/compiled_targets.md @@ -0,0 +1,63 @@ +# Compiled Targets + +- OK: `True` +- Targets: `4` +- Pass: `4` +- Warn: `0` +- Block: `0` + +## Target Transforms + +| Target | Status | Native Surface | Adapter Mode | Permissions | Degradation | Generated Files | +| --- | --- | --- | --- | --- | --- | --- | +| `openai` | `pass` | OpenAI-style interface metadata plus neutral Agent Skills source | `metadata-adapter` | `network, file_write, subprocess` | `metadata-adapter` | targets/openai/adapter.json, targets/openai/agents/openai.yaml | +| `claude` | `pass` | Claude-compatible neutral source folder with adapter notes | `neutral-source-plus-adapter` | `network, file_write, subprocess` | `neutral-source-plus-adapter` | targets/claude/adapter.json, targets/claude/README.md | +| `generic` | `pass` | Agent Skills compatible neutral package | `agent-skills-compatible` | `network, file_write, subprocess` | `neutral-source` | targets/generic/adapter.json | +| `agent-skills-compatible` | `pass` | Agent Skills standard source tree | `neutral-agent-skills-source` | `network, file_write, subprocess` | `neutral-source` | SKILL.md, agents/interface.yaml | + +## Native Behavior Contracts + +### openai + +- Native surface: OpenAI-style interface metadata plus neutral Agent Skills source +- Activation: Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata. +- Resources: Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file. +- Scripts: Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution. +- Permission enforcement: `metadata-only`; native enforcement `False` +- Review artifacts: targets/openai/agents/openai.yaml, targets/openai/adapter.json, reports/review-studio.html + +### claude + +- Native surface: Claude-compatible neutral source folder with adapter notes +- Activation: Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata. +- Resources: Preserve the source tree directly; write target notes in targets/claude/README.md. +- Scripts: Scripts remain local package resources and must be reviewed through trust and permission reports before use. +- Permission enforcement: `metadata-fallback`; native enforcement `False` +- Review artifacts: targets/claude/README.md, targets/claude/adapter.json, reports/review-studio.html + +### generic + +- Native surface: Agent Skills compatible neutral package +- Activation: Use SKILL.md name and description; consumers decide automatic or manual activation. +- Resources: Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources. +- Scripts: Expose script and permission metadata for downstream clients or installers to enforce. +- Permission enforcement: `consumer-enforced-or-metadata-only`; native enforcement `False` +- Review artifacts: targets/generic/adapter.json, reports/review-studio.html + +### agent-skills-compatible + +- Native surface: Agent Skills standard source tree +- Activation: Use SKILL.md frontmatter name and description for progressive disclosure. +- Resources: Keep optional directories as relative resources next to SKILL.md. +- Scripts: Scripts remain local optional resources and should advertise --help when executable. +- Permission enforcement: `consumer-enforced-or-metadata-only`; native enforcement `False` +- Review artifacts: SKILL.md, agents/interface.yaml, reports/review-studio.html + + +## Failures + +- None + +## Warnings + +- None diff --git a/reports/conformance_matrix.json b/reports/conformance_matrix.json index b51addc..20e9bce 100644 --- a/reports/conformance_matrix.json +++ b/reports/conformance_matrix.json @@ -32,6 +32,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -40,8 +41,7 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [] @@ -76,6 +76,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -84,8 +85,7 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [] @@ -120,6 +120,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -128,12 +129,11 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [ - "agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + "agent-skills uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0." ] }, { @@ -166,6 +166,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -174,12 +175,11 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [ - "vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + "vscode uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0." ] }, { @@ -212,6 +212,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -220,8 +221,7 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [] diff --git a/reports/conformance_matrix.md b/reports/conformance_matrix.md index d62a084..243e9bd 100644 --- a/reports/conformance_matrix.md +++ b/reports/conformance_matrix.md @@ -9,8 +9,8 @@ | --- | --- | --- | --- | | openai | pass | None | None | | claude | pass | None | None | -| agent-skills | pass | None | agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0. | -| vscode | pass | None | vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0. | +| agent-skills | pass | None | agent-skills uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0. | +| vscode | pass | None | vscode uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0. | | generic | pass | None | None | ## Reviewer Notes diff --git a/reports/context_budget.json b/reports/context_budget.json index 876de6e..4da126e 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -5,13 +5,13 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 734, - "other_text_tokens": 606468, - "estimated_initial_load_tokens": 910, - "estimated_total_text_tokens": 607202, - "relevant_file_count": 241, + "skill_body_tokens": 811, + "other_text_tokens": 864534, + "estimated_initial_load_tokens": 987, + "estimated_total_text_tokens": 865345, + "relevant_file_count": 316, "unused_resource_dirs": [], "quality_signal_points": 130, - "quality_density": 142.9 + "quality_density": 131.7 } } diff --git a/reports/context_budget.md b/reports/context_budget.md index 1da7709..9d678e5 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | -| root | `.` | `production` | 1000 | 910 | 734 | 142.9 | - | ok | +| root | `.` | `production` | 1000 | 987 | 811 | 131.7 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index efda026..ac17bd2 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -6,9 +6,9 @@ "path": ".", "budget_tier": "production", "budget_limit": 1000, - "initial_tokens": 910, - "skill_body_tokens": 734, - "quality_density": 142.9, + "initial_tokens": 987, + "skill_body_tokens": 811, + "quality_density": 131.7, "unused_resource_dirs": [], "ok": true, "warnings": [] diff --git a/reports/install_simulation.json b/reports/install_simulation.json new file mode 100644 index 0000000..facee86 --- /dev/null +++ b/reports/install_simulation.json @@ -0,0 +1,121 @@ +{ + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13", + "skill_dir": ".", + "package_dir": "dist", + "install_root": "dist/install-simulation/simulate-yao-meta-skill", + "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "summary": { + "archive_present": true, + "archive_entry_count": 483, + "archive_extracted": true, + "entrypoint_loaded": true, + "manifest_loaded": true, + "interface_loaded": true, + "adapter_count": 3, + "install_root_is_temp": false, + "failure_count": 0, + "warning_count": 0 + }, + "checks": [ + { + "id": "archive-present", + "status": "pass", + "detail": "Package archive exists: dist/yao-meta-skill.zip" + }, + { + "id": "archive-safe-paths", + "status": "pass", + "detail": "Archive has no absolute or parent-traversal entries" + }, + { + "id": "single-top-level", + "status": "pass", + "detail": "Archive top-level directory is yao-meta-skill" + }, + { + "id": "entrypoint-load", + "status": "pass", + "detail": "Installed SKILL.md frontmatter is readable" + }, + { + "id": "entrypoint-name", + "status": "pass", + "detail": "Installed SKILL.md name matches package directory" + }, + { + "id": "entrypoint-description", + "status": "pass", + "detail": "Installed SKILL.md description is present" + }, + { + "id": "manifest-load", + "status": "pass", + "detail": "Installed manifest.json is readable" + }, + { + "id": "manifest-name", + "status": "pass", + "detail": "Installed manifest name matches package manifest" + }, + { + "id": "manifest-version", + "status": "pass", + "detail": "Installed manifest version matches package manifest" + }, + { + "id": "interface-load", + "status": "pass", + "detail": "Installed agents/interface.yaml is readable" + }, + { + "id": "overview-report", + "status": "pass", + "detail": "Installed overview report is present" + }, + { + "id": "review-studio-report", + "status": "pass", + "detail": "Installed Review Studio report is present" + }, + { + "id": "adapter-claude", + "status": "pass", + "detail": "claude adapter is readable after package install simulation" + }, + { + "id": "adapter-claude-name", + "status": "pass", + "detail": "claude adapter name matches package manifest" + }, + { + "id": "adapter-generic", + "status": "pass", + "detail": "generic adapter is readable after package install simulation" + }, + { + "id": "adapter-generic-name", + "status": "pass", + "detail": "generic adapter name matches package manifest" + }, + { + "id": "adapter-openai", + "status": "pass", + "detail": "openai adapter is readable after package install simulation" + }, + { + "id": "adapter-openai-name", + "status": "pass", + "detail": "openai adapter name matches package manifest" + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "archive": "dist/yao-meta-skill.zip", + "package_manifest": "dist/manifest.json", + "json": "reports/install_simulation.json", + "markdown": "reports/install_simulation.md" + } +} diff --git a/reports/install_simulation.md b/reports/install_simulation.md new file mode 100644 index 0000000..7a06607 --- /dev/null +++ b/reports/install_simulation.md @@ -0,0 +1,42 @@ +# Install Simulation + +- OK: `True` +- Package directory: `dist` +- Archive extracted: `True` +- Entrypoint loaded: `True` +- Manifest loaded: `True` +- Interface loaded: `True` +- Adapters readable: `3` +- Failures: `0` +- Warnings: `0` + +## Checks + +| Check | Status | Detail | +| --- | --- | --- | +| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip | +| `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | +| `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill | +| `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable | +| `entrypoint-name` | `pass` | Installed SKILL.md name matches package directory | +| `entrypoint-description` | `pass` | Installed SKILL.md description is present | +| `manifest-load` | `pass` | Installed manifest.json is readable | +| `manifest-name` | `pass` | Installed manifest name matches package manifest | +| `manifest-version` | `pass` | Installed manifest version matches package manifest | +| `interface-load` | `pass` | Installed agents/interface.yaml is readable | +| `overview-report` | `pass` | Installed overview report is present | +| `review-studio-report` | `pass` | Installed Review Studio report is present | +| `adapter-claude` | `pass` | claude adapter is readable after package install simulation | +| `adapter-claude-name` | `pass` | claude adapter name matches package manifest | +| `adapter-generic` | `pass` | generic adapter is readable after package install simulation | +| `adapter-generic-name` | `pass` | generic adapter name matches package manifest | +| `adapter-openai` | `pass` | openai adapter is readable after package install simulation | +| `adapter-openai-name` | `pass` | openai adapter name matches package manifest | + +## Failures + +- None + +## Warnings + +- None diff --git a/reports/intent-confidence.json b/reports/intent-confidence.json index a6d252b..7401f59 100644 --- a/reports/intent-confidence.json +++ b/reports/intent-confidence.json @@ -1,79 +1,53 @@ { - "score": 30, - "band": "low", - "gate_passed": false, + "score": 100, + "band": "high", + "gate_passed": true, "strengths": [ - "The recurring job is concrete enough to anchor the package." + "The recurring job is concrete enough to anchor the package.", + "Real input shape is explicit.", + "The hand-back output is concrete.", + "Boundary exclusions are already explicit.", + "Operational constraints are visible." ], - "gaps": [ - { - "key": "real_inputs", - "label": "Real inputs are missing", - "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", - "severity": "high" - }, - { - "key": "primary_output", - "label": "Primary output is missing", - "reason": "The package does not yet know what it must hand back.", - "severity": "high" - }, - { - "key": "exclusions", - "label": "Near-neighbor exclusions are missing", - "reason": "The route may blur into nearby requests without an exclusion list.", - "severity": "high" - }, - { - "key": "constraints", - "label": "Constraints are missing", - "reason": "The package does not yet know which tradeoffs matter most.", - "severity": "high" - }, - { - "key": "standards", - "label": "Quality bar is implied, not explicit", - "reason": "The first evaluation target is still underspecified.", - "severity": "medium" - } - ], - "follow_up_questions": [ - { - "slot": "real_inputs", - "question": "What material will people actually hand to this skill in practice?", - "why": "Real input shape decides whether references, scripts, or examples are needed.", - "list": true, - "label": "Real inputs are missing", - "severity": "high" - }, - { - "slot": "primary_output", - "question": "What finished hand-back should this skill return so the next person can keep moving?", - "why": "The output is the anchor for package design and review.", - "list": false, - "label": "Primary output is missing", - "severity": "high" - }, - { - "slot": "exclusions", - "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?", - "why": "Exclusions are the fastest route to better trigger quality.", - "list": true, - "label": "Near-neighbor exclusions are missing", - "severity": "high" - } - ], - "anchor_sentence": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "gaps": [], + "follow_up_questions": [], + "anchor_sentence": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.", + "recommended_action": "Intent is clear enough to package the first routeable version.", "context": { - "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "real_inputs": [], - "primary_output": "", + "job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "real_inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "primary_output": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "exclusions": [], - "constraints": [], - "standards": [], - "correction": "", - "user_references": [] + "exclusions": [ + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill", + "raw private material that was not intentionally supplied as skill evidence", + "platform-specific plugin creation when the request is not about an agent skill package" + ], + "constraints": [ + "keep SKILL.md lean and route primarily through frontmatter description", + "put durable guidance in references, executable logic in scripts, and evidence in reports", + "default to the lightest reliable mode before adding governance weight", + "preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets", + "avoid raw prompt, output, transcript, or private content in telemetry" + ], + "standards": [ + "trigger boundaries must be tested with should-trigger and should-not-trigger cases", + "production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence", + "governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation", + "generated reports should be bilingual or reviewer-friendly when they are user-facing", + "each new asset must earn its place by reducing ambiguity, risk, or repeated work" + ], + "correction": "The 2.0 direction reframes this package from a Meta Skill factory into a Skill OS with Skill IR, compiler contracts, output eval, runtime conformance, trust, registry, Review Studio, Skill Atlas, and telemetry loops.", + "user_references": [ + "Yao Meta Skill 2.0 upgrade plan", + "Kami-style white report layout requirements for generated skill reports", + "local Waza engineering habits for review, design, diagnosis, and implementation discipline" + ] } } diff --git a/reports/intent-confidence.md b/reports/intent-confidence.md index 07c3a24..bab3942 100644 --- a/reports/intent-confidence.md +++ b/reports/intent-confidence.md @@ -1,31 +1,26 @@ # Intent Confidence -- Confidence score: `30/100` -- Confidence band: `low` -- Gate passed: `False` -- Recommended action: Pause before deep authoring and close the highest-leverage gaps first. +- Confidence score: `100/100` +- Confidence band: `high` +- Gate passed: `True` +- Recommended action: Intent is clear enough to package the first routeable version. ## Current Reading -Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package. ## Strong Signals - The recurring job is concrete enough to anchor the package. +- Real input shape is explicit. +- The hand-back output is concrete. +- Boundary exclusions are already explicit. +- Operational constraints are visible. ## Gaps To Close -- **Real inputs are missing** (`high`): Without real inputs, it is hard to choose assets, scripts, or examples. -- **Primary output is missing** (`high`): The package does not yet know what it must hand back. -- **Near-neighbor exclusions are missing** (`high`): The route may blur into nearby requests without an exclusion list. -- **Constraints are missing** (`high`): The package does not yet know which tradeoffs matter most. -- **Quality bar is implied, not explicit** (`medium`): The first evaluation target is still underspecified. +- No major intent gaps detected. ## Follow-Up Questions -- **What material will people actually hand to this skill in practice?** - - Why: Real input shape decides whether references, scripts, or examples are needed. -- **What finished hand-back should this skill return so the next person can keep moving?** - - Why: The output is the anchor for package design and review. -- **What nearby requests should this skill clearly leave out so the boundary stays clean?** - - Why: Exclusions are the fastest route to better trigger quality. +- No extra follow-up questions required before the first package. diff --git a/reports/intent-context.json b/reports/intent-context.json index 8fc3e49..492b5ae 100644 --- a/reports/intent-context.json +++ b/reports/intent-context.json @@ -1,11 +1,37 @@ { - "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "real_inputs": [], - "primary_output": "", + "job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "real_inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "primary_output": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "exclusions": [], - "constraints": [], - "standards": [], - "correction": "", - "user_references": [] + "exclusions": [ + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill", + "raw private material that was not intentionally supplied as skill evidence", + "platform-specific plugin creation when the request is not about an agent skill package" + ], + "constraints": [ + "keep SKILL.md lean and route primarily through frontmatter description", + "put durable guidance in references, executable logic in scripts, and evidence in reports", + "default to the lightest reliable mode before adding governance weight", + "preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets", + "avoid raw prompt, output, transcript, or private content in telemetry" + ], + "standards": [ + "trigger boundaries must be tested with should-trigger and should-not-trigger cases", + "production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence", + "governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation", + "generated reports should be bilingual or reviewer-friendly when they are user-facing", + "each new asset must earn its place by reducing ambiguity, risk, or repeated work" + ], + "correction": "The 2.0 direction reframes this package from a Meta Skill factory into a Skill OS with Skill IR, compiler contracts, output eval, runtime conformance, trust, registry, Review Studio, Skill Atlas, and telemetry loops.", + "user_references": [ + "Yao Meta Skill 2.0 upgrade plan", + "Kami-style white report layout requirements for generated skill reports", + "local Waza engineering habits for review, design, diagnosis, and implementation discipline" + ] } diff --git a/reports/output_blind_answer_key.json b/reports/output_blind_answer_key.json new file mode 100644 index 0000000..9de710d --- /dev/null +++ b/reports/output_blind_answer_key.json @@ -0,0 +1,56 @@ +{ + "schema_version": "1.0", + "seed": "yao-output-eval-blind-v1", + "summary": { + "pair_count": 5, + "with_skill_expected_count": 5, + "baseline_expected_count": 0 + }, + "answers": [ + { + "case_id": "skill-package-contract", + "variant_a_role": "with_skill", + "variant_b_role": "baseline", + "expected_winner_role": "with_skill", + "expected_winner_variant": "A", + "score_winner_role": "with_skill", + "delta": 100.0 + }, + { + "case_id": "output-eval-expectation", + "variant_a_role": "with_skill", + "variant_b_role": "baseline", + "expected_winner_role": "with_skill", + "expected_winner_variant": "A", + "score_winner_role": "with_skill", + "delta": 100.0 + }, + { + "case_id": "ir-before-packaging", + "variant_a_role": "baseline", + "variant_b_role": "with_skill", + "expected_winner_role": "with_skill", + "expected_winner_variant": "B", + "score_winner_role": "with_skill", + "delta": 100.0 + }, + { + "case_id": "near-neighbor-boundary", + "variant_a_role": "baseline", + "variant_b_role": "with_skill", + "expected_winner_role": "with_skill", + "expected_winner_variant": "B", + "score_winner_role": "with_skill", + "delta": 100.0 + }, + { + "case_id": "file-backed-governed-package", + "variant_a_role": "baseline", + "variant_b_role": "with_skill", + "expected_winner_role": "with_skill", + "expected_winner_variant": "B", + "score_winner_role": "with_skill", + "delta": 100.0 + } + ] +} diff --git a/reports/output_blind_review_pack.json b/reports/output_blind_review_pack.json new file mode 100644 index 0000000..27507ba --- /dev/null +++ b/reports/output_blind_review_pack.json @@ -0,0 +1,204 @@ +{ + "schema_version": "1.0", + "seed": "yao-output-eval-blind-v1", + "summary": { + "pair_count": 5, + "answer_key_separate": true, + "with_skill_hidden_count": 5 + }, + "pairs": [ + { + "case_id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "metadata": {}, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0 + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0 + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0 + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "skill-package-contract:A", + "output": "Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/." + }, + "variant_b": { + "blind_id": "skill-package-contract:B", + "output": "I can write a prompt for that workflow and include a short checklist." + } + }, + { + "case_id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "metadata": {}, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0 + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0 + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0 + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "output-eval-expectation:A", + "output": "Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md." + }, + "variant_b": { + "blind_id": "output-eval-expectation:B", + "output": "Add more examples and run the trigger eval again." + } + }, + { + "case_id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "metadata": {}, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0 + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0 + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0 + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "ir-before-packaging:A", + "output": "Export folders for each platform and adjust files as needed." + }, + "variant_b": { + "blind_id": "ir-before-packaging:B", + "output": "Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract." + } + }, + { + "case_id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "metadata": { + "case_type": "near_neighbor", + "tier": "production" + }, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0 + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0 + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "near-neighbor-boundary:A", + "output": "Create a SKILL.md and a reusable workflow anyway so future notes can use it." + }, + "variant_b": { + "blind_id": "near-neighbor-boundary:B", + "output": "Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract." + } + }, + { + "case_id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "metadata": { + "case_type": "boundary", + "tier": "governed" + }, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0 + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0 + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0 + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0 + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "file-backed-governed-package:A", + "output": "Draft a release announcement with the changelog, support notes, and owner name." + }, + "variant_b": { + "blind_id": "file-backed-governed-package:B", + "output": "Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them." + } + } + ] +} diff --git a/reports/output_blind_review_pack.md b/reports/output_blind_review_pack.md new file mode 100644 index 0000000..ba78e24 --- /dev/null +++ b/reports/output_blind_review_pack.md @@ -0,0 +1,97 @@ +# Output Blind A/B Review Pack + +This packet hides whether each variant came from the baseline or the skill-guided output. Use the separate answer key only after review. + +- Pairs: `5` +- Seed: `yao-output-eval-blind-v1` +- Answer key separate: `True` + +## Case: skill-package-contract + +Prompt: Turn this repeated workflow into a reusable team skill package. + +Rubric: +- `has-entrypoint` (1.0): Output names the routeable Skill entrypoint. +- `has-interface` (1.0): Output includes neutral interface metadata. +- `has-report` (1.0): Output points reviewer to generated overview report. +- `has-resource-boundary` (1.0): Output preserves entrypoint/resource separation. + +### Variant A + +Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/. + +### Variant B + +I can write a prompt for that workflow and include a short checklist. + +## Case: output-eval-expectation + +Prompt: Upgrade this production skill so we know whether the generated output is better than baseline. + +Rubric: +- `has-baseline-comparison` (1.0): Output explicitly compares with-skill and baseline outputs. +- `has-assertion-grading` (1.0): Output includes objective assertion grading. +- `has-scorecard` (1.0): Output produces a scorecard report path. +- `has-failure-taxonomy` (1.0): Output records failure taxonomy. + +### Variant A + +Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md. + +### Variant B + +Add more examples and run the trigger eval again. + +## Case: ir-before-packaging + +Prompt: Package this skill for OpenAI, Claude, Agent Skills, and generic targets. + +Rubric: +- `has-ir-first` (1.0): Output requires Skill IR before packaging. +- `has-core-ir-fields` (1.0): Output lists core IR contract fields. +- `has-targets` (1.0): Output names requested runtime targets. +- `has-semantic-preservation` (1.0): Output says targets must preserve the capability contract. + +### Variant A + +Export folders for each platform and adjust files as needed. + +### Variant B + +Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract. + +## Case: near-neighbor-boundary + +Prompt: I only need a one-off summary of these notes, not a reusable process. + +Rubric: +- `declines-skill` (1.0): Output refuses unnecessary skill creation for one-off work. +- `names-near-neighbor` (1.0): Output labels the request as near-neighbor instead of owned work. +- `requires-reuse-signal` (1.0): Output asks for repeat-use evidence before packaging. + +### Variant A + +Create a SKILL.md and a reusable workflow anyway so future notes can use it. + +### Variant B + +Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract. + +## Case: file-backed-governed-package + +Prompt: Turn the attached release brief source into a governed skill package. + +Rubric: +- `uses-file-backed-evidence` (1.0): Output names file-backed source evidence. +- `has-governance` (1.0): Output preserves governed ownership metadata. +- `has-output-contract` (1.0): Output preserves output and rollback boundaries. +- `has-trust-and-scorecard` (1.0): Output requires trust and output scorecard artifacts. +- `does-not-invent-evidence` (1.0): Output forbids invented launch evidence. + +### Variant A + +Draft a release announcement with the changelog, support notes, and owner name. + +### Variant B + +Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them. diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json new file mode 100644 index 0000000..32b0660 --- /dev/null +++ b/reports/output_execution_runs.json @@ -0,0 +1,291 @@ +{ + "schema_version": "1.0", + "ok": true, + "cases": "evals/output/cases.jsonl", + "runner": { + "mode": "command", + "command": [ + "python3", + "scripts/local_output_eval_runner.py" + ], + "timeout_seconds": 30.0 + }, + "summary": { + "case_count": 5, + "variant_run_count": 10, + "command_executed_count": 10, + "model_executed_count": 0, + "recorded_fixture_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "token_estimated_count": 10, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "failure_count": 0, + "gate_pass": true + }, + "runs": [ + { + "case_id": "skill-package-contract", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.22, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 16, + "output_tokens": 17, + "total_tokens": 33, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "failed_assertions": [ + "has-entrypoint", + "has-interface", + "has-report", + "has-resource-boundary" + ], + "output_sha256": "ece26a6edbbbea86a93faaf9f0eadf88efbae12f1c3ad81da9376cf2a889d101", + "failure": "" + }, + { + "case_id": "skill-package-contract", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.12, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 16, + "output_tokens": 57, + "total_tokens": 73, + "estimated": true + }, + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "8bee0131e17fdbe87e48457130ac6aeaa55ea30c919a45e77109852609f33eba", + "failure": "" + }, + { + "case_id": "output-eval-expectation", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.12, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 24, + "output_tokens": 12, + "total_tokens": 36, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "failed_assertions": [ + "has-baseline-comparison", + "has-assertion-grading", + "has-scorecard", + "has-failure-taxonomy" + ], + "output_sha256": "98fa332fb493b5dc14656a9d65a6c0a9c25ff61a14b71b647aad89cbdbd3ab35", + "failure": "" + }, + { + "case_id": "output-eval-expectation", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 30.03, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 24, + "output_tokens": 56, + "total_tokens": 80, + "estimated": true + }, + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "8efdbfefa6a36603682b7063b1753f9f939b3810c082d29686e297906d9f51c8", + "failure": "" + }, + { + "case_id": "ir-before-packaging", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.11, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 18, + "output_tokens": 15, + "total_tokens": 33, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "failed_assertions": [ + "has-ir-first", + "has-core-ir-fields", + "has-targets", + "has-semantic-preservation" + ], + "output_sha256": "07a4aa9ee9e48341d4b84d5cb9b6414901aa2c8841340d67447e187240c6c4c2", + "failure": "" + }, + { + "case_id": "ir-before-packaging", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 26.68, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 18, + "output_tokens": 62, + "total_tokens": 80, + "estimated": true + }, + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "707fde419b3d61e921561b1c4a8523e1f7b53b1acd5758a47ce21d9ed711fde0", + "failure": "" + }, + { + "case_id": "near-neighbor-boundary", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.07, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 19, + "total_tokens": 36, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 3, + "failed_assertions": [ + "declines-skill", + "names-near-neighbor", + "requires-reuse-signal" + ], + "output_sha256": "648ebaa6e09486d05f33fa6639dc973b029d115158588fd192efbe15ccaaeac4", + "failure": "" + }, + { + "case_id": "near-neighbor-boundary", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.57, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 48, + "total_tokens": 65, + "estimated": true + }, + "score": 100.0, + "passed_count": 3, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "17a3a5286ad573dea86e51f82c04b596babaecb92cde8879f09e7c519205e2e1", + "failure": "" + }, + { + "case_id": "file-backed-governed-package", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 29.2, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 20, + "total_tokens": 37, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 5, + "failed_assertions": [ + "uses-file-backed-evidence", + "has-governance", + "has-output-contract", + "has-trust-and-scorecard", + "does-not-invent-evidence" + ], + "output_sha256": "6e5827219459dee34143e4f11663415db80c3111291f75fb6c2115c7bda4cf30", + "failure": "" + }, + { + "case_id": "file-backed-governed-package", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 26.94, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 81, + "total_tokens": 98, + "estimated": true + }, + "score": 100.0, + "passed_count": 5, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "bcfdf036666d406abbc945fc42b93eb74d7074118893e540253c7d8715baad73", + "failure": "" + } + ], + "failures": [], + "artifacts": { + "json": "reports/output_execution_runs.json", + "markdown": "reports/output_execution_runs.md" + } +} diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md new file mode 100644 index 0000000..ac9d4c6 --- /dev/null +++ b/reports/output_execution_runs.md @@ -0,0 +1,41 @@ +# Output Execution Runs + +This report records how output-eval variants were produced and whether timing or token evidence is observed or estimated. + +- Cases: `5` +- Variant runs: `10` +- Command executed: `10` +- Model executed: `0` +- Recorded fixtures: `0` +- Timing observed: `10` +- Token observed: `0` +- Token estimated: `10` +- Delta: `100.0` +- Gate pass: `True` + +No model-executed runs are recorded yet. + +Use `--runner-command` with a provider-backed runner to replace recorded fixtures with real model output evidence. + +Command runner evidence is present. This proves the eval harness executed an external command, but it is not provider-backed model evidence unless the runner reports model metadata. + +## Runs + +| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | +| --- | --- | --- | --- | ---: | ---: | ---: | --- | +| skill-package-contract | baseline | command | local-output-eval-runner | 27.22 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 27.12 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 27.12 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 30.03 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 27.11 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.68 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.07 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.57 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 29.2 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.94 | 98 | 100.0 | pass | + +## Next Fixes + +- Keep recorded fixtures as reproducible baselines, but do not describe them as model-executed evidence. +- Add provider-backed command runners for holdout cases when release confidence depends on real generation behavior. +- Compare timing, token cost, and assertion deltas before promoting a skill to governed reuse. diff --git a/reports/output_quality_scorecard.json b/reports/output_quality_scorecard.json index bc47077..db0957e 100644 --- a/reports/output_quality_scorecard.json +++ b/reports/output_quality_scorecard.json @@ -11,7 +11,8 @@ "delta": 100.0, "regression_count": 0, "gate_pass": true, - "failure_taxonomy": [] + "failure_taxonomy": [], + "blind_pair_count": 5 }, "results": [ { @@ -800,8 +801,16 @@ } ], "failures": [], + "blind_review": { + "pack": "reports/output_blind_review_pack.json", + "answer_key": "reports/output_blind_answer_key.json", + "pair_count": 5 + }, "artifacts": { "json": "reports/output_quality_scorecard.json", - "markdown": "reports/output_quality_scorecard.md" + "markdown": "reports/output_quality_scorecard.md", + "blind_review_pack_json": "reports/output_blind_review_pack.json", + "blind_review_pack_md": "reports/output_blind_review_pack.md", + "blind_answer_key_json": "reports/output_blind_answer_key.json" } } diff --git a/reports/output_quality_scorecard.md b/reports/output_quality_scorecard.md index aa1cc12..20a32ad 100644 --- a/reports/output_quality_scorecard.md +++ b/reports/output_quality_scorecard.md @@ -7,8 +7,12 @@ This v0 scorecard compares static without-skill and with-skill outputs using ass - With-skill pass rate: `100.0` - Delta: `100.0` - Regressions: `0` +- Blind A/B pairs: `5` - Gate pass: `True` +Blind review artifacts are generated separately so reviewers can inspect A/B outputs without seeing the answer key. +Run output review adjudication after reviewer decisions are recorded; pending cases should stay pending rather than being counted as human agreement. + ## Case Results | Case | Baseline | With Skill | Delta | Winner | Failed With-Skill Assertions | diff --git a/reports/output_review_adjudication.json b/reports/output_review_adjudication.json new file mode 100644 index 0000000..0d58d32 --- /dev/null +++ b/reports/output_review_adjudication.json @@ -0,0 +1,73 @@ +{ + "schema_version": "1.0", + "ok": true, + "summary": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "agreement_count": 0, + "disagreement_count": 0, + "invalid_decision_count": 0, + "agreement_rate": null, + "needs_review": true, + "failure_count": 0 + }, + "reviewer": "", + "reviewed_at": "", + "artifacts": { + "blind_pack": "reports/output_blind_review_pack.json", + "answer_key": "reports/output_blind_answer_key.json", + "decisions": "reports/output_review_decisions.json", + "json": "reports/output_review_adjudication.json", + "markdown": "reports/output_review_adjudication.md" + }, + "template_written": false, + "pairs": [ + { + "case_id": "skill-package-contract", + "status": "pending", + "expected_winner_variant": "A", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Turn this repeated workflow into a reusable team skill package." + }, + { + "case_id": "output-eval-expectation", + "status": "pending", + "expected_winner_variant": "A", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline." + }, + { + "case_id": "ir-before-packaging", + "status": "pending", + "expected_winner_variant": "B", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets." + }, + { + "case_id": "near-neighbor-boundary", + "status": "pending", + "expected_winner_variant": "B", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "I only need a one-off summary of these notes, not a reusable process." + }, + { + "case_id": "file-backed-governed-package", + "status": "pending", + "expected_winner_variant": "B", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Turn the attached release brief source into a governed skill package." + } + ], + "failures": [] +} diff --git a/reports/output_review_adjudication.md b/reports/output_review_adjudication.md new file mode 100644 index 0000000..6c12de4 --- /dev/null +++ b/reports/output_review_adjudication.md @@ -0,0 +1,29 @@ +# Output Review Adjudication + +This report adjudicates reviewer choices from the blind A/B output review pack against the separate answer key. + +- Pairs: `5` +- Judgments: `0` +- Pending: `5` +- Agreement rate: `n/a` +- Invalid decisions: `0` + +No reviewer decisions recorded yet. + +Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication. + +## Case Adjudication + +| Case | Reviewer | Expected | Status | Confidence | Reason | +| --- | --- | --- | --- | ---: | --- | +| skill-package-contract | pending | A | pending | | | +| output-eval-expectation | pending | A | pending | | | +| ir-before-packaging | pending | B | pending | | | +| near-neighbor-boundary | pending | B | pending | | | +| file-backed-governed-package | pending | B | pending | | | + +## Next Fixes + +- Keep the blind review pack separate from the answer key until decisions are recorded. +- Treat disagreement cases as prompts for rubric tuning or output improvement. +- Add model-executed holdout runs after this human adjudication harness is stable. diff --git a/reports/package_verification.json b/reports/package_verification.json new file mode 100644 index 0000000..17be2de --- /dev/null +++ b/reports/package_verification.json @@ -0,0 +1,516 @@ +{ + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13", + "skill_dir": ".", + "package_dir": "dist", + "summary": { + "target_count": 3, + "adapter_count": 3, + "archive_present": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "archive_entry_count": 483, + "failure_count": 0, + "warning_count": 0 + }, + "checks": [ + { + "id": "package-manifest", + "status": "pass", + "detail": "Package manifest exists: dist/manifest.json" + }, + { + "id": "openai-adapter", + "status": "pass", + "detail": "Adapter exists for target: openai" + }, + { + "id": "openai-field-name", + "status": "pass", + "detail": "openai adapter includes field: name" + }, + { + "id": "openai-field-description", + "status": "pass", + "detail": "openai adapter includes field: description" + }, + { + "id": "openai-field-version", + "status": "pass", + "detail": "openai adapter includes field: version" + }, + { + "id": "openai-field-display_name", + "status": "pass", + "detail": "openai adapter includes field: display_name" + }, + { + "id": "openai-field-short_description", + "status": "pass", + "detail": "openai adapter includes field: short_description" + }, + { + "id": "openai-field-default_prompt", + "status": "pass", + "detail": "openai adapter includes field: default_prompt" + }, + { + "id": "openai-field-job_to_be_done", + "status": "pass", + "detail": "openai adapter includes field: job_to_be_done" + }, + { + "id": "openai-field-ir_source", + "status": "pass", + "detail": "openai adapter includes field: ir_source" + }, + { + "id": "openai-field-ir_schema_version", + "status": "pass", + "detail": "openai adapter includes field: ir_schema_version" + }, + { + "id": "openai-field-semantic_contract", + "status": "pass", + "detail": "openai adapter includes field: semantic_contract" + }, + { + "id": "openai-field-semantic_parity", + "status": "pass", + "detail": "openai adapter includes field: semantic_parity" + }, + { + "id": "openai-field-compiler", + "status": "pass", + "detail": "openai adapter includes field: compiler" + }, + { + "id": "openai-field-compiled_contract", + "status": "pass", + "detail": "openai adapter includes field: compiled_contract" + }, + { + "id": "openai-field-permission_contract", + "status": "pass", + "detail": "openai adapter includes field: permission_contract" + }, + { + "id": "openai-field-target_permission_contract", + "status": "pass", + "detail": "openai adapter includes field: target_permission_contract" + }, + { + "id": "openai-field-target_native_contract", + "status": "pass", + "detail": "openai adapter includes field: target_native_contract" + }, + { + "id": "openai-field-target_transform", + "status": "pass", + "detail": "openai adapter includes field: target_transform" + }, + { + "id": "openai-field-canonical_metadata", + "status": "pass", + "detail": "openai adapter includes field: canonical_metadata" + }, + { + "id": "openai-field-canonical_format", + "status": "pass", + "detail": "openai adapter includes field: canonical_format" + }, + { + "id": "openai-field-activation_mode", + "status": "pass", + "detail": "openai adapter includes field: activation_mode" + }, + { + "id": "openai-field-execution_context", + "status": "pass", + "detail": "openai adapter includes field: execution_context" + }, + { + "id": "openai-field-shell", + "status": "pass", + "detail": "openai adapter includes field: shell" + }, + { + "id": "openai-field-trust_level", + "status": "pass", + "detail": "openai adapter includes field: trust_level" + }, + { + "id": "openai-field-remote_inline_execution", + "status": "pass", + "detail": "openai adapter includes field: remote_inline_execution" + }, + { + "id": "openai-field-degradation_strategy", + "status": "pass", + "detail": "openai adapter includes field: degradation_strategy" + }, + { + "id": "openai-field-portability_profile", + "status": "pass", + "detail": "openai adapter includes field: portability_profile" + }, + { + "id": "claude-adapter", + "status": "pass", + "detail": "Adapter exists for target: claude" + }, + { + "id": "claude-field-name", + "status": "pass", + "detail": "claude adapter includes field: name" + }, + { + "id": "claude-field-description", + "status": "pass", + "detail": "claude adapter includes field: description" + }, + { + "id": "claude-field-version", + "status": "pass", + "detail": "claude adapter includes field: version" + }, + { + "id": "claude-field-display_name", + "status": "pass", + "detail": "claude adapter includes field: display_name" + }, + { + "id": "claude-field-short_description", + "status": "pass", + "detail": "claude adapter includes field: short_description" + }, + { + "id": "claude-field-default_prompt", + "status": "pass", + "detail": "claude adapter includes field: default_prompt" + }, + { + "id": "claude-field-job_to_be_done", + "status": "pass", + "detail": "claude adapter includes field: job_to_be_done" + }, + { + "id": "claude-field-ir_source", + "status": "pass", + "detail": "claude adapter includes field: ir_source" + }, + { + "id": "claude-field-ir_schema_version", + "status": "pass", + "detail": "claude adapter includes field: ir_schema_version" + }, + { + "id": "claude-field-semantic_contract", + "status": "pass", + "detail": "claude adapter includes field: semantic_contract" + }, + { + "id": "claude-field-semantic_parity", + "status": "pass", + "detail": "claude adapter includes field: semantic_parity" + }, + { + "id": "claude-field-compiler", + "status": "pass", + "detail": "claude adapter includes field: compiler" + }, + { + "id": "claude-field-compiled_contract", + "status": "pass", + "detail": "claude adapter includes field: compiled_contract" + }, + { + "id": "claude-field-permission_contract", + "status": "pass", + "detail": "claude adapter includes field: permission_contract" + }, + { + "id": "claude-field-target_permission_contract", + "status": "pass", + "detail": "claude adapter includes field: target_permission_contract" + }, + { + "id": "claude-field-target_native_contract", + "status": "pass", + "detail": "claude adapter includes field: target_native_contract" + }, + { + "id": "claude-field-target_transform", + "status": "pass", + "detail": "claude adapter includes field: target_transform" + }, + { + "id": "claude-field-canonical_metadata", + "status": "pass", + "detail": "claude adapter includes field: canonical_metadata" + }, + { + "id": "claude-field-canonical_format", + "status": "pass", + "detail": "claude adapter includes field: canonical_format" + }, + { + "id": "claude-field-activation_mode", + "status": "pass", + "detail": "claude adapter includes field: activation_mode" + }, + { + "id": "claude-field-execution_context", + "status": "pass", + "detail": "claude adapter includes field: execution_context" + }, + { + "id": "claude-field-shell", + "status": "pass", + "detail": "claude adapter includes field: shell" + }, + { + "id": "claude-field-trust_level", + "status": "pass", + "detail": "claude adapter includes field: trust_level" + }, + { + "id": "claude-field-remote_inline_execution", + "status": "pass", + "detail": "claude adapter includes field: remote_inline_execution" + }, + { + "id": "claude-field-degradation_strategy", + "status": "pass", + "detail": "claude adapter includes field: degradation_strategy" + }, + { + "id": "claude-field-portability_profile", + "status": "pass", + "detail": "claude adapter includes field: portability_profile" + }, + { + "id": "generic-adapter", + "status": "pass", + "detail": "Adapter exists for target: generic" + }, + { + "id": "generic-field-name", + "status": "pass", + "detail": "generic adapter includes field: name" + }, + { + "id": "generic-field-description", + "status": "pass", + "detail": "generic adapter includes field: description" + }, + { + "id": "generic-field-version", + "status": "pass", + "detail": "generic adapter includes field: version" + }, + { + "id": "generic-field-display_name", + "status": "pass", + "detail": "generic adapter includes field: display_name" + }, + { + "id": "generic-field-short_description", + "status": "pass", + "detail": "generic adapter includes field: short_description" + }, + { + "id": "generic-field-default_prompt", + "status": "pass", + "detail": "generic adapter includes field: default_prompt" + }, + { + "id": "generic-field-job_to_be_done", + "status": "pass", + "detail": "generic adapter includes field: job_to_be_done" + }, + { + "id": "generic-field-ir_source", + "status": "pass", + "detail": "generic adapter includes field: ir_source" + }, + { + "id": "generic-field-ir_schema_version", + "status": "pass", + "detail": "generic adapter includes field: ir_schema_version" + }, + { + "id": "generic-field-semantic_contract", + "status": "pass", + "detail": "generic adapter includes field: semantic_contract" + }, + { + "id": "generic-field-semantic_parity", + "status": "pass", + "detail": "generic adapter includes field: semantic_parity" + }, + { + "id": "generic-field-compiler", + "status": "pass", + "detail": "generic adapter includes field: compiler" + }, + { + "id": "generic-field-compiled_contract", + "status": "pass", + "detail": "generic adapter includes field: compiled_contract" + }, + { + "id": "generic-field-permission_contract", + "status": "pass", + "detail": "generic adapter includes field: permission_contract" + }, + { + "id": "generic-field-target_permission_contract", + "status": "pass", + "detail": "generic adapter includes field: target_permission_contract" + }, + { + "id": "generic-field-target_native_contract", + "status": "pass", + "detail": "generic adapter includes field: target_native_contract" + }, + { + "id": "generic-field-target_transform", + "status": "pass", + "detail": "generic adapter includes field: target_transform" + }, + { + "id": "generic-field-canonical_metadata", + "status": "pass", + "detail": "generic adapter includes field: canonical_metadata" + }, + { + "id": "generic-field-canonical_format", + "status": "pass", + "detail": "generic adapter includes field: canonical_format" + }, + { + "id": "generic-field-activation_mode", + "status": "pass", + "detail": "generic adapter includes field: activation_mode" + }, + { + "id": "generic-field-execution_context", + "status": "pass", + "detail": "generic adapter includes field: execution_context" + }, + { + "id": "generic-field-shell", + "status": "pass", + "detail": "generic adapter includes field: shell" + }, + { + "id": "generic-field-trust_level", + "status": "pass", + "detail": "generic adapter includes field: trust_level" + }, + { + "id": "generic-field-remote_inline_execution", + "status": "pass", + "detail": "generic adapter includes field: remote_inline_execution" + }, + { + "id": "generic-field-degradation_strategy", + "status": "pass", + "detail": "generic adapter includes field: degradation_strategy" + }, + { + "id": "generic-field-portability_profile", + "status": "pass", + "detail": "generic adapter includes field: portability_profile" + }, + { + "id": "openai-file-targets/openai/adapter.json", + "status": "pass", + "detail": "Package contains targets/openai/adapter.json" + }, + { + "id": "openai-file-targets/openai/agents/openai.yaml", + "status": "pass", + "detail": "Package contains targets/openai/agents/openai.yaml" + }, + { + "id": "claude-file-targets/claude/adapter.json", + "status": "pass", + "detail": "Package contains targets/claude/adapter.json" + }, + { + "id": "claude-file-targets/claude/README.md", + "status": "pass", + "detail": "Package contains targets/claude/README.md" + }, + { + "id": "generic-file-targets/generic/adapter.json", + "status": "pass", + "detail": "Package contains targets/generic/adapter.json" + }, + { + "id": "archive-safe-paths", + "status": "pass", + "detail": "Archive has no absolute or parent-traversal entries" + }, + { + "id": "archive-entry-yao-meta-skill/SKILL.md", + "status": "pass", + "detail": "Archive contains yao-meta-skill/SKILL.md" + }, + { + "id": "archive-entry-yao-meta-skill/manifest.json", + "status": "pass", + "detail": "Archive contains yao-meta-skill/manifest.json" + }, + { + "id": "archive-entry-yao-meta-skill/agents/interface.yaml", + "status": "pass", + "detail": "Archive contains yao-meta-skill/agents/interface.yaml" + }, + { + "id": "archive-excludes-generated", + "status": "pass", + "detail": "Archive excludes generated dist/ and tests/tmp* contents" + }, + { + "id": "registry-ok", + "status": "pass", + "detail": "Registry audit is OK" + }, + { + "id": "registry-name-match", + "status": "pass", + "detail": "Registry package name matches package manifest" + }, + { + "id": "registry-version-match", + "status": "pass", + "detail": "Registry package version matches package manifest" + }, + { + "id": "registry-compat-openai", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: openai" + }, + { + "id": "registry-compat-claude", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: claude" + }, + { + "id": "registry-compat-generic", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: generic" + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "manifest": "dist/manifest.json", + "archive": "dist/yao-meta-skill.zip", + "json": "reports/package_verification.json", + "markdown": "reports/package_verification.md" + } +} diff --git a/reports/package_verification.md b/reports/package_verification.md new file mode 100644 index 0000000..c17f69c --- /dev/null +++ b/reports/package_verification.md @@ -0,0 +1,120 @@ +# Package Verification + +- OK: `True` +- Package directory: `dist` +- Targets: `3 / 3` adapters present +- Archive present: `True` +- Archive SHA256: `4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06` +- Failures: `0` +- Warnings: `0` + +## Checks + +| Check | Status | Detail | +| --- | --- | --- | +| `package-manifest` | `pass` | Package manifest exists: dist/manifest.json | +| `openai-adapter` | `pass` | Adapter exists for target: openai | +| `openai-field-name` | `pass` | openai adapter includes field: name | +| `openai-field-description` | `pass` | openai adapter includes field: description | +| `openai-field-version` | `pass` | openai adapter includes field: version | +| `openai-field-display_name` | `pass` | openai adapter includes field: display_name | +| `openai-field-short_description` | `pass` | openai adapter includes field: short_description | +| `openai-field-default_prompt` | `pass` | openai adapter includes field: default_prompt | +| `openai-field-job_to_be_done` | `pass` | openai adapter includes field: job_to_be_done | +| `openai-field-ir_source` | `pass` | openai adapter includes field: ir_source | +| `openai-field-ir_schema_version` | `pass` | openai adapter includes field: ir_schema_version | +| `openai-field-semantic_contract` | `pass` | openai adapter includes field: semantic_contract | +| `openai-field-semantic_parity` | `pass` | openai adapter includes field: semantic_parity | +| `openai-field-compiler` | `pass` | openai adapter includes field: compiler | +| `openai-field-compiled_contract` | `pass` | openai adapter includes field: compiled_contract | +| `openai-field-permission_contract` | `pass` | openai adapter includes field: permission_contract | +| `openai-field-target_permission_contract` | `pass` | openai adapter includes field: target_permission_contract | +| `openai-field-target_native_contract` | `pass` | openai adapter includes field: target_native_contract | +| `openai-field-target_transform` | `pass` | openai adapter includes field: target_transform | +| `openai-field-canonical_metadata` | `pass` | openai adapter includes field: canonical_metadata | +| `openai-field-canonical_format` | `pass` | openai adapter includes field: canonical_format | +| `openai-field-activation_mode` | `pass` | openai adapter includes field: activation_mode | +| `openai-field-execution_context` | `pass` | openai adapter includes field: execution_context | +| `openai-field-shell` | `pass` | openai adapter includes field: shell | +| `openai-field-trust_level` | `pass` | openai adapter includes field: trust_level | +| `openai-field-remote_inline_execution` | `pass` | openai adapter includes field: remote_inline_execution | +| `openai-field-degradation_strategy` | `pass` | openai adapter includes field: degradation_strategy | +| `openai-field-portability_profile` | `pass` | openai adapter includes field: portability_profile | +| `claude-adapter` | `pass` | Adapter exists for target: claude | +| `claude-field-name` | `pass` | claude adapter includes field: name | +| `claude-field-description` | `pass` | claude adapter includes field: description | +| `claude-field-version` | `pass` | claude adapter includes field: version | +| `claude-field-display_name` | `pass` | claude adapter includes field: display_name | +| `claude-field-short_description` | `pass` | claude adapter includes field: short_description | +| `claude-field-default_prompt` | `pass` | claude adapter includes field: default_prompt | +| `claude-field-job_to_be_done` | `pass` | claude adapter includes field: job_to_be_done | +| `claude-field-ir_source` | `pass` | claude adapter includes field: ir_source | +| `claude-field-ir_schema_version` | `pass` | claude adapter includes field: ir_schema_version | +| `claude-field-semantic_contract` | `pass` | claude adapter includes field: semantic_contract | +| `claude-field-semantic_parity` | `pass` | claude adapter includes field: semantic_parity | +| `claude-field-compiler` | `pass` | claude adapter includes field: compiler | +| `claude-field-compiled_contract` | `pass` | claude adapter includes field: compiled_contract | +| `claude-field-permission_contract` | `pass` | claude adapter includes field: permission_contract | +| `claude-field-target_permission_contract` | `pass` | claude adapter includes field: target_permission_contract | +| `claude-field-target_native_contract` | `pass` | claude adapter includes field: target_native_contract | +| `claude-field-target_transform` | `pass` | claude adapter includes field: target_transform | +| `claude-field-canonical_metadata` | `pass` | claude adapter includes field: canonical_metadata | +| `claude-field-canonical_format` | `pass` | claude adapter includes field: canonical_format | +| `claude-field-activation_mode` | `pass` | claude adapter includes field: activation_mode | +| `claude-field-execution_context` | `pass` | claude adapter includes field: execution_context | +| `claude-field-shell` | `pass` | claude adapter includes field: shell | +| `claude-field-trust_level` | `pass` | claude adapter includes field: trust_level | +| `claude-field-remote_inline_execution` | `pass` | claude adapter includes field: remote_inline_execution | +| `claude-field-degradation_strategy` | `pass` | claude adapter includes field: degradation_strategy | +| `claude-field-portability_profile` | `pass` | claude adapter includes field: portability_profile | +| `generic-adapter` | `pass` | Adapter exists for target: generic | +| `generic-field-name` | `pass` | generic adapter includes field: name | +| `generic-field-description` | `pass` | generic adapter includes field: description | +| `generic-field-version` | `pass` | generic adapter includes field: version | +| `generic-field-display_name` | `pass` | generic adapter includes field: display_name | +| `generic-field-short_description` | `pass` | generic adapter includes field: short_description | +| `generic-field-default_prompt` | `pass` | generic adapter includes field: default_prompt | +| `generic-field-job_to_be_done` | `pass` | generic adapter includes field: job_to_be_done | +| `generic-field-ir_source` | `pass` | generic adapter includes field: ir_source | +| `generic-field-ir_schema_version` | `pass` | generic adapter includes field: ir_schema_version | +| `generic-field-semantic_contract` | `pass` | generic adapter includes field: semantic_contract | +| `generic-field-semantic_parity` | `pass` | generic adapter includes field: semantic_parity | +| `generic-field-compiler` | `pass` | generic adapter includes field: compiler | +| `generic-field-compiled_contract` | `pass` | generic adapter includes field: compiled_contract | +| `generic-field-permission_contract` | `pass` | generic adapter includes field: permission_contract | +| `generic-field-target_permission_contract` | `pass` | generic adapter includes field: target_permission_contract | +| `generic-field-target_native_contract` | `pass` | generic adapter includes field: target_native_contract | +| `generic-field-target_transform` | `pass` | generic adapter includes field: target_transform | +| `generic-field-canonical_metadata` | `pass` | generic adapter includes field: canonical_metadata | +| `generic-field-canonical_format` | `pass` | generic adapter includes field: canonical_format | +| `generic-field-activation_mode` | `pass` | generic adapter includes field: activation_mode | +| `generic-field-execution_context` | `pass` | generic adapter includes field: execution_context | +| `generic-field-shell` | `pass` | generic adapter includes field: shell | +| `generic-field-trust_level` | `pass` | generic adapter includes field: trust_level | +| `generic-field-remote_inline_execution` | `pass` | generic adapter includes field: remote_inline_execution | +| `generic-field-degradation_strategy` | `pass` | generic adapter includes field: degradation_strategy | +| `generic-field-portability_profile` | `pass` | generic adapter includes field: portability_profile | +| `openai-file-targets/openai/adapter.json` | `pass` | Package contains targets/openai/adapter.json | +| `openai-file-targets/openai/agents/openai.yaml` | `pass` | Package contains targets/openai/agents/openai.yaml | +| `claude-file-targets/claude/adapter.json` | `pass` | Package contains targets/claude/adapter.json | +| `claude-file-targets/claude/README.md` | `pass` | Package contains targets/claude/README.md | +| `generic-file-targets/generic/adapter.json` | `pass` | Package contains targets/generic/adapter.json | +| `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | +| `archive-entry-yao-meta-skill/SKILL.md` | `pass` | Archive contains yao-meta-skill/SKILL.md | +| `archive-entry-yao-meta-skill/manifest.json` | `pass` | Archive contains yao-meta-skill/manifest.json | +| `archive-entry-yao-meta-skill/agents/interface.yaml` | `pass` | Archive contains yao-meta-skill/agents/interface.yaml | +| `archive-excludes-generated` | `pass` | Archive excludes generated dist/ and tests/tmp* contents | +| `registry-ok` | `pass` | Registry audit is OK | +| `registry-name-match` | `pass` | Registry package name matches package manifest | +| `registry-version-match` | `pass` | Registry package version matches package manifest | +| `registry-compat-openai` | `pass` | Registry compatibility is reviewable for target: openai | +| `registry-compat-claude` | `pass` | Registry compatibility is reviewable for target: claude | +| `registry-compat-generic` | `pass` | Registry compatibility is reviewable for target: generic | + +## Failures + +- None + +## Warnings + +- None diff --git a/reports/prompt-quality-profile.json b/reports/prompt-quality-profile.json index 3c84c0d..8af0544 100644 --- a/reports/prompt-quality-profile.json +++ b/reports/prompt-quality-profile.json @@ -2,14 +2,26 @@ "skill_name": "yao-meta-skill", "relevance": "prompt-heavy", "primary_task_family": { - "key": "dialogue_interaction", - "label": "Dialogue interaction", + "key": "execution_operation", + "label": "Execution operation", "matched_keywords": [ - "dialogue", - "conversation" + "workflow", + "runbook" ] }, "task_families": [ + { + "key": "execution_operation", + "label": "Execution operation", + "score": 2, + "matched_keywords": [ + "workflow", + "runbook" + ], + "role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", + "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", + "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." + }, { "key": "dialogue_interaction", "label": "Dialogue interaction", @@ -23,15 +35,15 @@ "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." }, { - "key": "execution_operation", - "label": "Execution operation", + "key": "creative_generation", + "label": "Creative generation", "score": 1, "matched_keywords": [ - "workflow" + "content" ], - "role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", - "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", - "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." + "role_guidance": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.", + "task_guidance": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.", + "format_guidance": "Return options with rationale, selection criteria, and refinement paths." }, { "key": "prompt_engineering", @@ -46,27 +58,27 @@ } ], "complexity": { - "band": "complex", - "score": 5, - "reason": "multiple inputs, constraints, or task families require tradeoff handling" + "band": "expert", + "score": 27, + "reason": "multiple task families plus governance, evaluation, or expert-level constraints" }, "need_model": { - "explicit_need": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "explicit_need": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "implicit_need": "The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.", - "scenario": "not yet explicit", + "scenario": "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions, an existing skill directory that needs refactor, evaluation, packaging, or governance hardening, target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution, benchmark references, local constraints, desired maturity tier, and review standards", "user_level": "infer from examples and standards; ask only if it changes output depth", - "success_standard": "usable output with clear validation cues" + "success_standard": "trigger boundaries must be tested with should-trigger and should-not-trigger cases, production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence, governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation, generated reports should be bilingual or reviewer-friendly when they are user-facing, each new asset must earn its place by reducing ambiguity, risk, or repeated work" }, "rtf_to_skill": { - "role": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", - "task": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", - "format": "Return concise prompts, decision points, and reviewer-visible assumptions." + "role": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", + "task": "Convert the job into ordered steps with validation checks and stop conditions.", + "format": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." }, "quality_matrix": [ { "key": "completeness", "label": "Completeness", - "score": 80, + "score": 100, "matched_signals": [ "output", "constraint", @@ -77,8 +89,10 @@ { "key": "clarity", "label": "Clarity", - "score": 80, - "matched_signals": [], + "score": 85, + "matched_signals": [ + "clear" + ], "repair": "Replace broad verbs with observable actions and define what done means." }, { @@ -104,12 +118,12 @@ { "key": "specificity", "label": "Specificity", - "score": 70, + "score": 80, "matched_signals": [], "repair": "Anchor wording in the user's audience, domain nouns, and target outcome." } ], - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "self_repair_checks": [ "Check explicit need, implicit need, scenario, user level, and success standard before deepening.", "Map Role, Task, and Format into skill behavior, not decorative prompt labels.", diff --git a/reports/prompt-quality-profile.md b/reports/prompt-quality-profile.md index 1e9579b..a061130 100644 --- a/reports/prompt-quality-profile.md +++ b/reports/prompt-quality-profile.md @@ -2,41 +2,41 @@ Skill: `yao-meta-skill` Relevance: `prompt-heavy` -Overall quality score: `82.0/100` +Overall quality score: `89.0/100` ## Primary Task Family -**Dialogue interaction** -- Matched keywords: dialogue, conversation +**Execution operation** +- Matched keywords: workflow, runbook ## Complexity -- Band: `complex` -- Score: `5` -- Reason: multiple inputs, constraints, or task families require tradeoff handling +- Band: `expert` +- Score: `27` +- Reason: multiple task families plus governance, evaluation, or expert-level constraints ## Need Model -- Explicit Need: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +- Explicit Need: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. - Implicit Need: The reusable skill needs a stable role, task, and output contract rather than a one-off prompt. -- Scenario: not yet explicit +- Scenario: rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions, an existing skill directory that needs refactor, evaluation, packaging, or governance hardening, target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution, benchmark references, local constraints, desired maturity tier, and review standards - User Level: infer from examples and standards; ask only if it changes output depth -- Success Standard: usable output with clear validation cues +- Success Standard: trigger boundaries must be tested with should-trigger and should-not-trigger cases, production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence, governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation, generated reports should be bilingual or reviewer-friendly when they are user-facing, each new asset must earn its place by reducing ambiguity, risk, or repeated work ## RTF To Skill Mapping -- Role: Use a conversational role that asks only high-leverage questions and remembers the user's goal. -- Task: Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list. -- Format: Return concise prompts, decision points, and reviewer-visible assumptions. +- Role: Use an operator role with explicit boundaries, inputs, outputs, and failure handling. +- Task: Convert the job into ordered steps with validation checks and stop conditions. +- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant. ## Quality Matrix -### Completeness — 80/100 +### Completeness — 100/100 - Matched signals: output, constraint, standard - Repair: Name missing inputs, outputs, constraints, or success standards before deepening the package. -### Clarity — 80/100 -- Matched signals: none +### Clarity — 85/100 +- Matched signals: clear - Repair: Replace broad verbs with observable actions and define what done means. ### Consistency — 90/100 @@ -47,12 +47,19 @@ Overall quality score: `82.0/100` - Matched signals: use, workflow - Repair: Add runnable steps, examples, or verification cues instead of abstract advice. -### Specificity — 70/100 +### Specificity — 80/100 - Matched signals: none - Repair: Anchor wording in the user's audience, domain nouns, and target outcome. ## Matched Task Families +### Execution operation +- Score: `2` +- Keywords: workflow, runbook +- Role: Use an operator role with explicit boundaries, inputs, outputs, and failure handling. +- Task: Convert the job into ordered steps with validation checks and stop conditions. +- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant. + ### Dialogue interaction - Score: `2` - Keywords: dialogue, conversation @@ -60,12 +67,12 @@ Overall quality score: `82.0/100` - Task: Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list. - Format: Return concise prompts, decision points, and reviewer-visible assumptions. -### Execution operation +### Creative generation - Score: `1` -- Keywords: workflow -- Role: Use an operator role with explicit boundaries, inputs, outputs, and failure handling. -- Task: Convert the job into ordered steps with validation checks and stop conditions. -- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant. +- Keywords: content +- Role: Use a taste-aware creator role with clear audience, tone, and originality boundaries. +- Task: Generate variants, explain selection logic, and preserve the user's distinctive constraints. +- Format: Return options with rationale, selection criteria, and refinement paths. ### Prompt engineering - Score: `1` diff --git a/reports/reference-synthesis.json b/reports/reference-synthesis.json index 03d6dad..a6141d2 100644 --- a/reports/reference-synthesis.json +++ b/reports/reference-synthesis.json @@ -2,9 +2,9 @@ "skill_name": "yao-meta-skill", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "intent_confidence": { - "score": 30, - "band": "low", - "gate_passed": false + "score": 100, + "band": "high", + "gate_passed": true }, "github_benchmarks": [ { @@ -38,49 +38,51 @@ "source_tracks": [ { "source_type": "official", - "name": "Official workflow product ergonomics", + "name": "Official skill anatomy and context discipline", "evidence_mode": "curated-pattern-track", "matched_keywords": [ - "workflow" + "metadata", + "references" ], - "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", - "why_relevant": "This track matches: workflow." + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "why_relevant": "This track matches: metadata, references." }, { "source_type": "research", - "name": "Hypothesis-test-learn loop", + "name": "Human-in-the-loop verification", "evidence_mode": "curated-pattern-track", "matched_keywords": [ - "general fit" + "review", + "govern" ], - "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", - "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", - "why_relevant": "This track matches: general fit." + "borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.", + "avoid": "Do not force every skill through heavyweight review when the risk is low.", + "why_relevant": "This track matches: review, govern." }, { "source_type": "principles", "name": "Boundary-first design", "evidence_mode": "curated-pattern-track", "matched_keywords": [ - "general fit" + "route" ], "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", "avoid": "Do not expand execution assets until route boundaries stay clean.", - "why_relevant": "This track matches: general fit." + "why_relevant": "This track matches: route." } ], "synthesis": { "borrow_now": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed.", "Borrow the discipline of defining what the skill should not own before growing the package.", "Borrow the way it turns a messy workflow into a repeatable operating path.", "Borrow the clear execution entrypoints and command structure." ], "avoid_now": [ - "Do not mimic product polish that adds UI bulk without improving clarity.", - "Do not create experimental overhead that exceeds the skill's real risk tier.", + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not force every skill through heavyweight review when the risk is low.", "Do not expand execution assets until route boundaries stay clean.", "Do not import process overhead that only exists for that project's scale.", "Do not copy repo-specific commands or environment assumptions verbatim." @@ -96,10 +98,50 @@ "source_count": 6, "accepted": [ { - "name": "Official workflow product ergonomics", + "name": "Official skill anatomy and context discipline", "source_type": "official", - "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", + "borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "avoid": "Do not let packaging or platform concerns swallow the core job boundary.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + }, + { + "name": "Human-in-the-loop verification", + "source_type": "research", + "borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.", + "avoid": "Do not force every skill through heavyweight review when the risk is low.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + }, + { + "name": "Boundary-first design", + "source_type": "principles", + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", "gates": { "recurrence": true, "generativity": true, @@ -157,48 +199,6 @@ } ], "deferred": [ - { - "name": "Hypothesis-test-learn loop", - "source_type": "research", - "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", - "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", - "gates": { - "recurrence": true, - "generativity": true, - "distinctiveness": false, - "boundary": true - }, - "passed": [ - "recurrence", - "generativity", - "boundary" - ], - "missing": [ - "distinctiveness" - ], - "score": 3 - }, - { - "name": "Boundary-first design", - "source_type": "principles", - "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", - "avoid": "Do not expand execution assets until route boundaries stay clean.", - "gates": { - "recurrence": true, - "generativity": true, - "distinctiveness": false, - "boundary": true - }, - "passed": [ - "recurrence", - "generativity", - "boundary" - ], - "missing": [ - "distinctiveness" - ], - "score": 3 - }, { "name": "multica-ai/andrej-karpathy-skills", "source_type": "github", @@ -221,27 +221,34 @@ "score": 3 } ], - "summary": "3 accepted, 3 deferred using threshold 4/4." + "summary": "5 accepted, 1 deferred using threshold 4/4." }, - "conflicts": [], + "conflicts": [ + { + "key": "lightweight_vs_governance", + "summary": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", + "user_preference": "lightweight or speed-first", + "benchmark_pressure": "governance or evaluation-heavy patterns" + } + ], "recommendation": { - "summary": "Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.", + "summary": "Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary.", "borrow_now": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed." + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed." ], "avoid_for_now": [ - "Do not mimic product polish that adds UI bulk without improving clarity.", - "Do not create experimental overhead that exceeds the skill's real risk tier." + "Do not let packaging or platform concerns swallow the core job boundary.", + "Do not force every skill through heavyweight review when the risk is low." ], - "why": "Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package.", + "why": "There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.", "user_decision_required": true }, "visibility": { "mode": "explicit", "user_decision_required": true, "reasons": [ - "intent_uncertain" + "design_conflict" ], "user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.", "reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers." diff --git a/reports/reference-synthesis.md b/reports/reference-synthesis.md index 8ade5ff..77adb18 100644 --- a/reports/reference-synthesis.md +++ b/reports/reference-synthesis.md @@ -2,7 +2,7 @@ Skill: `yao-meta-skill` - Description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. -- Intent confidence: `30/100` (`low`) +- Intent confidence: `100/100` (`high`) ## Live GitHub Benchmarks @@ -26,72 +26,72 @@ Skill: `yao-meta-skill` ## Curated World-Class Pattern Tracks -### Official workflow product ergonomics +### Official skill anatomy and context discipline - Type: `official` - Evidence mode: `curated-pattern-track` -- Why relevant: This track matches: workflow. -- Borrow: Borrow a first-time operator flow that explains itself before it asks for more structure. -- Avoid: Do not mimic product polish that adds UI bulk without improving clarity. +- Why relevant: This track matches: metadata, references. +- Borrow: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. +- Avoid: Do not let packaging or platform concerns swallow the core job boundary. -### Hypothesis-test-learn loop +### Human-in-the-loop verification - Type: `research` - Evidence mode: `curated-pattern-track` -- Why relevant: This track matches: general fit. -- Borrow: Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. -- Avoid: Do not create experimental overhead that exceeds the skill's real risk tier. +- Why relevant: This track matches: review, govern. +- Borrow: Borrow a review checkpoint wherever trust matters more than raw speed. +- Avoid: Do not force every skill through heavyweight review when the risk is low. ### Boundary-first design - Type: `principles` - Evidence mode: `curated-pattern-track` -- Why relevant: This track matches: general fit. +- Why relevant: This track matches: route. - Borrow: Borrow the discipline of defining what the skill should not own before growing the package. - Avoid: Do not expand execution assets until route boundaries stay clean. ## Borrow Now -- Borrow a first-time operator flow that explains itself before it asks for more structure. -- Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. +- Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. +- Borrow a review checkpoint wherever trust matters more than raw speed. - Borrow the discipline of defining what the skill should not own before growing the package. - Borrow the way it turns a messy workflow into a repeatable operating path. - Borrow the clear execution entrypoints and command structure. ## Avoid Now -- Do not mimic product polish that adds UI bulk without improving clarity. -- Do not create experimental overhead that exceeds the skill's real risk tier. +- Do not let packaging or platform concerns swallow the core job boundary. +- Do not force every skill through heavyweight review when the risk is low. - Do not expand execution assets until route boundaries stay clean. - Do not import process overhead that only exists for that project's scale. - Do not copy repo-specific commands or environment assumptions verbatim. ## Pattern Gate -- Summary: 3 accepted, 3 deferred using threshold 4/4. +- Summary: 5 accepted, 1 deferred using threshold 4/4. - Acceptance threshold: `4/4` - Accepted patterns: - - **Official workflow product ergonomics**: 4/4 (recurrence, generativity, distinctiveness, boundary) + - **Official skill anatomy and context discipline**: 4/4 (recurrence, generativity, distinctiveness, boundary) + - **Human-in-the-loop verification**: 4/4 (recurrence, generativity, distinctiveness, boundary) + - **Boundary-first design**: 4/4 (recurrence, generativity, distinctiveness, boundary) - **obra/superpowers**: 4/4 (recurrence, generativity, distinctiveness, boundary) - **affaan-m/ECC**: 4/4 (recurrence, generativity, distinctiveness, boundary) - Deferred patterns: - - **Hypothesis-test-learn loop**: missing distinctiveness - - **Boundary-first design**: missing distinctiveness - **multica-ai/andrej-karpathy-skills**: missing generativity ## Default Recommendation -- Summary: Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity. -- Why: Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package. +- Summary: Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary. +- Why: There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure. - User decision required: `True` ## Visibility Mode - Mode: `explicit` -- Reasons: intent_uncertain +- Reasons: design_conflict - User note: Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call. - Reviewer note: Keep the full benchmark and synthesis evidence visible for authors and reviewers. ## Conflict Check -- No material design conflict detected. Keep the synthesis silent for the user. +- **lightweight_vs_governance**: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure. ## Quality Lift Thesis diff --git a/reports/registry_audit.json b/reports/registry_audit.json new file mode 100644 index 0000000..6f6525d --- /dev/null +++ b/reports/registry_audit.json @@ -0,0 +1,91 @@ +{ + "ok": true, + "schema_version": "2.0", + "skill_dir": ".", + "registry_dir": "registry", + "package": { + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe", + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "agent-skills": "pass", + "vscode": "pass", + "generic": "pass", + "agent-skills-compatible": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "package_verification": "reports/package_verification.json", + "install_simulated": true, + "install_simulation": "reports/install_simulation.json" + }, + "generated_at": "2026-06-13" + }, + "index": { + "schema_version": "2.0", + "generated_at": "2026-06-13", + "package_count": 1, + "packages": [ + { + "name": "yao-meta-skill", + "version": "1.1.0", + "maturity": "governed", + "owner": "Yao Team", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "package_metadata": "registry/packages/yao-meta-skill.json", + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" + } + ] + }, + "failures": [], + "warnings": [], + "artifacts": { + "index": "registry/index.json", + "package": "registry/packages/yao-meta-skill.json", + "json": "reports/registry_audit.json", + "markdown": "reports/registry_audit.md" + } +} diff --git a/reports/registry_audit.md b/reports/registry_audit.md new file mode 100644 index 0000000..6468aa9 --- /dev/null +++ b/reports/registry_audit.md @@ -0,0 +1,37 @@ +# Registry Audit + +- OK: `True` +- Package: `yao-meta-skill` +- Version: `1.1.0` +- Maturity: `governed` +- Owner: `Yao Team` +- License: `MIT` +- Package SHA256: `9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe` +- Archive SHA256: `4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06` +- Install simulated: `True` + +## Compatibility + +| Target | Status | +| --- | --- | +| `openai` | `pass` | +| `claude` | `pass` | +| `agent-skills` | `pass` | +| `vscode` | `pass` | +| `generic` | `pass` | +| `agent-skills-compatible` | `pass` | + +## Failures + +- None + +## Warnings + +- None + +## Artifacts + +- index: `registry/index.json` +- package: `registry/packages/yao-meta-skill.json` +- json: `reports/registry_audit.json` +- markdown: `reports/registry_audit.md` diff --git a/reports/review-studio.html b/reports/review-studio.html index 20c8194..15be5d2 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -116,8 +116,89 @@ gap: 3px; } .muted { color: var(--muted); } + .actions-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + } + .annotations-grid { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + } + .action-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 10px; + min-width: 0; + } + .action-card.warn { border-left: 4px solid var(--warn); } + .action-card.block { border-left: 4px solid var(--block); } + .annotation-card { + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 10px; + min-width: 0; + } + .annotation-card.warning { border-left: 4px solid var(--warn); } + .annotation-card.blocker { border-left: 4px solid var(--block); } + .annotation-card.resolved { opacity: .72; } + .action-card span, + .annotation-card span, + .action-card small, + .annotation-card small, + .action-card footer, + .annotation-card footer, + .action-card dd { + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + } + .annotation-card dd { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; } + .action-card dl, .annotation-card dl { + display: grid; + grid-template-columns: 80px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + } + .action-card dt, .annotation-card dt { color: var(--ink); font-size: 14px; } + .action-card dd, .annotation-card dd { margin: 0; } + .source-ref-list { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 6px; + } + .source-ref-list li { + display: grid; + gap: 2px; + min-width: 0; + padding: 8px 0 0; + border-top: 1px solid var(--line); + } + .source-ref-list a, + .source-ref-list span { + overflow-wrap: anywhere; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + } + .source-ref-list small { + font-size: 12px; + color: var(--muted); + } + code { + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + } @media (max-width: 980px) { - .metrics, .gates, .twocol { grid-template-columns: 1fr; } + .metrics, .gates, .twocol, .actions-grid, .annotations-grid { grid-template-columns: 1fr; } main { padding: 32px 18px 60px; } nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; } nav a { flex: 0 0 auto; } @@ -125,7 +206,7 @@ - +
Review Studio 2.0
@@ -133,65 +214,121 @@

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

审查结论 - review - Score 87/100 + ready + Score 100/100

核心指标

-
Skill IR2.0.0

4 targets in platform-neutral contract

Output Delta100.0

5 cases; 1 file-backed

Runtime5/5

target conformance pass rate

Trust0

50 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

+
Skill IR2.0.0

4 targets in platform-neutral contract

Compiler4/4

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe3/3

0 native enforcement targets

Trust0

62 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

4 targets; MIT license

Archivepass

483 zip entries; package verification

Installpass

3 adapters readable; local install simulation

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
关注

意图画布

intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
通过

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 910/1000; quality density 142.9

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
关注

信任报告

0 secrets; 50 scripts; 2 network-capable scripts

reports/security_trust_report.json 证据
关注

组合治理

12 skills; 5 route collisions; 9 owner gaps; 10 stale

reports/skill_atlas.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked

reports/promotion_decisions.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
通过

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 987/1000; quality density 131.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 62 scripts; 2 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

3/3 targets probed; native 0; metadata fallback 3; residual risks 3

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
通过

人工批准

0 active waivers cover current warnings

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据

阻断事项

无。

-

关注事项

  • 意图画布intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.
  • 信任报告0 secrets; 50 scripts; 2 network-capable scripts
  • 组合治理12 skills; 5 route collisions; 9 owner gaps; 10 stale
+

关注事项

无。

+
+

修复动作

+

当前没有 blocker 或 warning。保持现有证据链即可。

+
+ +
+

审查批注

+

当前批注：0 条批注；开放 0；阻断 0

+

当前没有 reviewer 批注。

+
+

意图画布

-

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+

Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.

证据路径

-
  • skill_overviewreports/skill-overview.html
  • review_viewerreports/review-viewer.html
  • output_evalreports/output_quality_scorecard.md
  • runtime_conformancereports/conformance_matrix.md
  • trust_reportreports/security_trust_report.md
  • skill_atlasreports/skill_atlas.html
  • migrationdocs/migration-v2.md
+
  • skill_overviewreports/skill-overview.html
  • review_viewerreports/review-viewer.html
  • output_evalreports/output_quality_scorecard.md
  • output_executionreports/output_execution_runs.md
  • output_blind_reviewreports/output_blind_review_pack.md
  • output_review_adjudicationreports/output_review_adjudication.md
  • runtime_conformancereports/conformance_matrix.md
  • trust_reportreports/security_trust_report.md
  • permission_policysecurity/permission_policy.md
  • runtime_permissionsreports/runtime_permission_probes.md
  • skill_atlasreports/skill_atlas.html
  • compiled_targetsreports/compiled_targets.md
  • adoption_driftreports/adoption_drift_report.md
  • review_waiversreports/review_waivers.md
  • review_annotationsreports/review_annotations.md
  • registry_auditreports/registry_audit.md
  • package_verificationreports/package_verification.md
  • install_simulationreports/install_simulation.md
  • upgrade_checkreports/upgrade_check.md
  • migrationdocs/migration-v2.md

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

-

组合治理

{'skill_count': 12, 'route_collision_count': 5, 'owner_gap_count': 9, 'stale_count': 10, 'shared_resource_count': 0, 'no_route_opportunity_count': 0}

+

组合治理

{'skill_count': 12, 'actionable_skill_count': 1, 'route_collision_count': 5, 'actionable_route_collision_count': 0, 'owner_gap_count': 9, 'actionable_owner_gap_count': 0, 'stale_count': 10, 'actionable_stale_count': 0, 'shared_resource_count': 0, 'no_route_opportunity_count': 0, 'non_actionable_issue_count': 24}

-

输出实验

{'case_count': 5, 'file_backed_case_count': 1, 'near_neighbor_case_count': 1, 'boundary_case_count': 1, 'baseline_pass_rate': 0.0, 'with_skill_pass_rate': 100.0, 'delta': 100.0, 'regression_count': 0, 'gate_pass': True, 'failure_taxonomy': []}

-

发布标准

Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor 和 boundary case。

+

输出实验

{'case_count': 5, 'file_backed_case_count': 1, 'near_neighbor_case_count': 1, 'boundary_case_count': 1, 'baseline_pass_rate': 0.0, 'with_skill_pass_rate': 100.0, 'delta': 100.0, 'regression_count': 0, 'gate_pass': True, 'failure_taxonomy': [], 'blind_pair_count': 5}

+

执行证据

{'case_count': 5, 'variant_run_count': 10, 'command_executed_count': 10, 'model_executed_count': 0, 'recorded_fixture_count': 0, 'timing_observed_count': 10, 'token_observed_count': 0, 'token_estimated_count': 10, 'baseline_pass_rate': 0.0, 'with_skill_pass_rate': 100.0, 'delta': 100.0, 'regression_count': 0, 'failure_count': 0, 'gate_pass': True}

+
+ +
+

盲评包

{'pair_count': 5, 'answer_key_separate': True, 'with_skill_hidden_count': 5}

+

审定报告

{'pair_count': 5, 'judgment_count': 0, 'pending_count': 5, 'agreement_count': 0, 'disagreement_count': 0, 'invalid_decision_count': 0, 'agreement_rate': None, 'needs_review': True, 'failure_count': 0}

+
+ +
+

发布标准

Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。

+

人工结论

没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。

+
+ +
+

评审方式

先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

+

运行方式

reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。

运行矩阵

{'target_count': 5, 'pass_count': 5, 'fail_count': 0}

-

上下文

initial load 910/1000; quality density 142.9

+

目标编译

{'target_count': 4, 'pass_count': 4, 'warn_count': 0, 'block_count': 0, 'failure_count': 0, 'warning_count': 0}

+
+ +
+

上下文

initial load 987/1000; quality density 131.7

+

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

{'scanned_files': 127, 'script_count': 50, 'secret_findings': 0, 'dependency_files': ['requirements-ci.txt'], 'network_script_count': 2, 'interactive_script_count': 0, 'package_sha256': '18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad'}

+

信任报告

{'scanned_files': 146, 'script_count': 62, 'internal_module_count': 3, 'secret_findings': 0, 'dependency_files': ['requirements-ci.txt'], 'network_script_count': 2, 'network_policy_covered_count': 2, 'network_policy_missing_count': 0, 'file_write_script_count': 47, 'permission_required_count': 3, 'permission_approved_count': 3, 'permission_missing_count': 0, 'permission_invalid_count': 0, 'permission_expired_count': 0, 'help_smoke_checked_count': 59, 'help_smoke_failed_count': 0, 'interactive_script_count': 0, 'package_hash_scope': 'source-contract-without-generated-reports', 'package_hash_file_count': 146, 'package_sha256': '9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe'}

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

+
+

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

+

批准策略

高权限能力需要 reviewer、scope、reason、expires_at 和 openai/claude/generic 目标端 enforcement 说明。

+
+ +
+

权限探针

3/3 targets probed; native 0; metadata fallback 3; residual risks 3

+

运行边界

{'target_count': 3, 'pass_count': 3, 'fail_count': 0, 'native_enforcement_count': 0, 'metadata_fallback_count': 3, 'residual_risk_count': 3, 'required_capability_count': 3, 'failure_count': 0}

+
+
-

组合治理

12 skills; 5 route collisions; 9 owner gaps; 10 stale

-

下一动作

优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再做 registry 或 telemetry。

+

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

+

下一动作

优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再用运营回路判断真实影响。

+
+ +
+

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

+

漂移信号

{'event_count': 1, 'activation_count': 1, 'accepted_count': 1, 'edited_count': 0, 'rejected_count': 0, 'missed_count': 0, 'failed_count': 0, 'adoption_rate': 100.0, 'missed_trigger_count': 0, 'wrong_trigger_count': 0, 'bad_output_count': 0, 'script_error_count': 0, 'missing_resource_count': 0, 'review_overdue_count': 0, 'risk_band': 'low', 'event_types': {'skill_activation': 1}, 'failure_types': {}}

+
+ +
+

人工批准

0 active waivers cover current warnings

+

批准台账

{'waiver_count': 0, 'active_count': 0, 'expired_count': 0, 'invalid_count': 0, 'covered_gate_count': 0, 'covered_gate_keys': []}

+
+ +
+

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters

+

包体元数据

{'schema_version': '2.0', 'name': 'yao-meta-skill', 'version': '1.1.0', 'description': 'Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.', 'targets': ['openai', 'claude', 'generic', 'agent-skills-compatible'], 'maturity': 'governed', 'owner': 'Yao Team', 'review_cadence': 'quarterly', 'trust_level': 'local', 'license': 'MIT', 'checksums': {'package_sha256': '9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe', 'archive_sha256': '4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06'}, 'compatibility': {'openai': 'pass', 'claude': 'pass', 'agent-skills': 'pass', 'vscode': 'pass', 'generic': 'pass', 'agent-skills-compatible': 'pass'}, 'source': {'skill_dir': '.', 'skill_ir': 'skill-ir/examples/yao-meta-skill.json', 'ir_schema_version': '2.0.0', 'canonical_metadata': 'agents/interface.yaml'}, 'artifacts': {'overview_html': 'reports/skill-overview.html', 'review_studio_html': 'reports/review-studio.html', 'trust_report': 'reports/security_trust_report.md', 'conformance_matrix': 'reports/conformance_matrix.md', 'compiled_targets': 'reports/compiled_targets.md', 'package_verification': 'reports/package_verification.md', 'install_simulation': 'reports/install_simulation.md', 'adoption_drift': 'reports/adoption_drift_report.md', 'review_waivers': 'reports/review_waivers.md', 'review_annotations': 'reports/review_annotations.md', 'package_metadata': 'registry/packages/yao-meta-skill.json'}, 'distribution': {'archive_verified': True, 'archive_sha256': '4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06', 'package_verification': 'reports/package_verification.json', 'install_simulated': True, 'install_simulation': 'reports/install_simulation.json'}, 'generated_at': '2026-06-13'}

-

发布路线

0 promote; 3 keep current; 0 blocked

-

世界级缺口

下一阶段应继续推进 IR-first compiler、registry audit、telemetry drift loop，以及更严格的 governed trust gates。

+

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

+

包体验证

{'target_count': 3, 'adapter_count': 3, 'archive_present': True, 'archive_sha256': '4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06', 'archive_entry_count': 483, 'failure_count': 0, 'warning_count': 0}

diff --git a/reports/review-studio.json b/reports/review-studio.json index a7ee125..2ee0bca 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -3,18 +3,23 @@ "ok": true, "skill_dir": ".", "summary": { - "decision": "review", - "world_class_score": 87, - "gate_count": 8, + "decision": "ready", + "world_class_score": 100, + "gate_count": 13, "blocker_count": 0, - "warning_count": 3 + "warning_count": 0, + "action_count": 0, + "annotation_count": 0, + "open_annotation_count": 0, + "open_annotation_blocker_count": 0, + "open_annotation_warning_count": 0 }, "gates": [ { "key": "intent-canvas", "label": "意图画布", - "status": "warn", - "detail": "intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.", + "status": "pass", + "detail": "intent confidence 100/100; Intent is clear enough to package the first routeable version.", "evidence": "reports/intent-confidence.json", "link": "intent-confidence.md" }, @@ -30,7 +35,7 @@ "key": "output-lab", "label": "输出实验", "status": "pass", - "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5", "evidence": "reports/output_quality_scorecard.json", "link": "output_quality_scorecard.md" }, @@ -38,7 +43,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 910/1000; quality density 142.9", + "detail": "initial load 987/1000; quality density 131.7", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -53,62 +58,91 @@ { "key": "trust-report", "label": "信任报告", - "status": "warn", - "detail": "0 secrets; 50 scripts; 2 network-capable scripts", + "status": "pass", + "detail": "0 secrets; 62 scripts; 2 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, + { + "key": "permission-gates", + "label": "权限批准", + "status": "pass", + "detail": "3/3 permissions approved; gaps 0; required file_write, network, subprocess", + "evidence": "reports/security_trust_report.json + security/permission_policy.json", + "link": "../security/permission_policy.md" + }, + { + "key": "permission-runtime", + "label": "权限探针", + "status": "pass", + "detail": "3/3 targets probed; native 0; metadata fallback 3; residual risks 3", + "evidence": "reports/runtime_permission_probes.json", + "link": "runtime_permission_probes.md" + }, { "key": "skill-atlas", "label": "组合治理", - "status": "warn", - "detail": "12 skills; 5 route collisions; 9 owner gaps; 10 stale", + "status": "pass", + "detail": "12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues", "evidence": "reports/skill_atlas.json", "link": "skill_atlas.html" }, + { + "key": "operations-loop", + "label": "运营回路", + "status": "pass", + "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low", + "evidence": "reports/adoption_drift_report.json", + "link": "adoption_drift_report.md" + }, + { + "key": "review-waivers", + "label": "人工批准", + "status": "pass", + "detail": "0 active waivers cover current warnings", + "evidence": "reports/review_waivers.json", + "link": "review_waivers.md" + }, + { + "key": "registry-audit", + "label": "注册审计", + "status": "pass", + "detail": "yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters", + "evidence": "reports/registry_audit.json + reports/install_simulation.json", + "link": "registry_audit.md" + }, { "key": "release-notes", "label": "发布路线", "status": "pass", - "detail": "0 promote; 3 keep current; 0 blocked", - "evidence": "reports/promotion_decisions.json + docs/migration-v2.md", + "detail": "0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended", + "evidence": "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", "link": "promotion_decisions.md" } ], "blockers": [], - "warnings": [ - { - "key": "intent-canvas", - "label": "意图画布", - "status": "warn", - "detail": "intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.", - "evidence": "reports/intent-confidence.json", - "link": "intent-confidence.md" - }, - { - "key": "trust-report", - "label": "信任报告", - "status": "warn", - "detail": "0 secrets; 50 scripts; 2 network-capable scripts", - "evidence": "reports/security_trust_report.json", - "link": "security_trust_report.md" - }, - { - "key": "skill-atlas", - "label": "组合治理", - "status": "warn", - "detail": "12 skills; 5 route collisions; 9 owner gaps; 10 stale", - "evidence": "reports/skill_atlas.json", - "link": "skill_atlas.html" - } - ], + "warnings": [], + "review_actions": [], "evidence_paths": { "skill_overview": "reports/skill-overview.html", "review_viewer": "reports/review-viewer.html", "output_eval": "reports/output_quality_scorecard.md", + "output_execution": "reports/output_execution_runs.md", + "output_blind_review": "reports/output_blind_review_pack.md", + "output_review_adjudication": "reports/output_review_adjudication.md", "runtime_conformance": "reports/conformance_matrix.md", "trust_report": "reports/security_trust_report.md", + "permission_policy": "security/permission_policy.md", + "runtime_permissions": "reports/runtime_permission_probes.md", "skill_atlas": "reports/skill_atlas.html", + "compiled_targets": "reports/compiled_targets.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "registry_audit": "reports/registry_audit.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "upgrade_check": "reports/upgrade_check.md", "migration": "docs/migration-v2.md" }, "data": { @@ -126,10 +160,23 @@ "SKILL.md", "agents/interface.yaml", "reports/skill-ir.json", + "reports/compiled_targets.md", "reports/output_quality_scorecard.md", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.md", "reports/conformance_matrix.md", "reports/security_trust_report.md", + "reports/runtime_permission_probes.md", "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/review_annotations.md", "reports/review-studio.html", "reports/skill-overview.html" ], @@ -166,17 +213,17 @@ "label": "证据充分", "score": 100, "reasons": [ - "已生成 13 / 13 类报告证据。", + "已生成 20 / 20 类报告证据。", "skill-ir.json 已存在。", - "intent-dialogue.json 已存在。", - "intent-confidence.json 已存在。" + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" ] }, "maintainability_score": { "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 346 个词/字。", + "SKILL.md 约 365 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -197,7 +244,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 346 个词/字，references 约 11313 个词/字。", + "入口约 365 个词/字，references 约 14247 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -205,7 +252,7 @@ }, "capability_profile": { "archetype": "governed", - "task_family": "Dialogue interaction", + "task_family": "Execution operation", "maturity": "governed", "trigger_strength": "手动触发 + description 路由", "reuse_scope": "跨平台", @@ -218,7 +265,7 @@ "nodes": [ { "title": "意图澄清", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." }, { "title": "边界路由", @@ -238,7 +285,7 @@ }, { "title": "杠杆升级", - "body": "Ask one focused question and update intent context before adding assets." + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." } ], "layers": [ @@ -257,13 +304,13 @@ "shell": "bash" }, "inputs": [ - "用户提供的工作流、提示词、文档、记录或散乱笔记", - "期望沉淀的复用场景、排除项、约束和质量标准" + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" ], "outputs": [ - "可路由的 SKILL.md", - "agents/interface.yaml 元数据", - "必要的 references、scripts、evals、reports 证据", + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "结构化 Skill 目录，共 8 类关键资产。" ], "should_trigger": [ @@ -295,13 +342,13 @@ "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", - "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", - "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", - "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。" + "上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -309,7 +356,7 @@ "每次升级后重新生成报告并检查分数原因。" ], "artifact_design": { - "design_system": "review studio", + "design_system": "metric editorial", "highlights": [ "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", "Name the working directory and required inputs for commands.", @@ -318,24 +365,24 @@ ] }, "prompt_quality": { - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "highlights": [ - "Primary prompt task family: Dialogue interaction.", - "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", - "Completeness: 80/100.", - "Clarity: 80/100." + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." ] }, "system_model": { "stability": { - "score": 86, - "band": "stable-first-pass" + "score": 100, + "band": "system-ready" }, "highlights": [ - "Stability: stable-first-pass (86/100).", - "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", - "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." ] } }, @@ -352,21 +399,21 @@ "name": "输出漂移风险", "impact": 2, "probability": 1, - "signal": "已生成 13 / 13 类报告证据。", + "signal": "已生成 20 / 20 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "证据不足风险", "impact": 3, "probability": 1, - "signal": "已生成 13 / 13 类报告证据。", + "signal": "已生成 20 / 20 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 346 个词/字。", + "signal": "SKILL.md 约 365 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -528,13 +575,13 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 29 + "file_count": 32 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 50 + "file_count": 62 }, { "path": "evals", @@ -546,10 +593,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 144 + "file_count": 170 } ], - "file_count": 255, + "file_count": 296, "folder_count": 4, "distribution": [ { @@ -570,11 +617,11 @@ }, { "label": "references", - "value": 29 + "value": 32 }, { "label": "scripts", - "value": 50 + "value": 62 }, { "label": "evals", @@ -582,7 +629,7 @@ }, { "label": "reports", - "value": 144 + "value": 170 } ] }, @@ -697,13 +744,13 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 29 + "file_count": 32 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 50 + "file_count": 62 }, { "path": "evals", @@ -715,16 +762,16 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 144 + "file_count": 170 } ], "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", - "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", - "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", - "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "scenarios": [ "把重复流程沉淀为可复用的 agent skill。", @@ -740,20 +787,20 @@ }, "io_contract": { "inputs": [ - "用户提供的工作流、提示词、文档、记录或散乱笔记", - "期望沉淀的复用场景、排除项、约束和质量标准" + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" ], "outputs": [ - "可路由的 SKILL.md", - "agents/interface.yaml 元数据", - "必要的 references、scripts、evals、reports 证据", + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "结构化 Skill 目录，共 8 类关键资产。" ] }, "principles": [ { "title": "意图澄清", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." }, { "title": "边界路由", @@ -773,7 +820,7 @@ }, { "title": "杠杆升级", - "body": "Ask one focused question and update intent context before adding assets." + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." } ], "roadmap": [ @@ -821,6 +868,81 @@ "trigger_samples": 8, "output_eval_cases": 3 }, + "compiled_targets": { + "ok": true, + "schema_version": "1.0", + "summary": { + "target_count": 4, + "pass_count": 4, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter_mode": "metadata-adapter", + "degradation_strategy": "metadata-adapter", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "permission_enforcement": "metadata-only", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "adapter_mode": "neutral-source-plus-adapter", + "degradation_strategy": "neutral-source-plus-adapter", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "permission_enforcement": "metadata-fallback", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [] + }, + { + "target": "generic", + "status": "pass", + "adapter_mode": "agent-skills-compatible", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills compatible neutral package", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "targets/generic/adapter.json" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "agent-skills-compatible", + "status": "pass", + "adapter_mode": "neutral-agent-skills-source", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills standard source tree", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "unsupported_features": [], + "warnings": [] + } + ], + "failures": [], + "warnings": [] + }, "output_quality": { "case_count": 5, "file_backed_case_count": 1, @@ -831,39 +953,417 @@ "delta": 100.0, "regression_count": 0, "gate_pass": true, - "failure_taxonomy": [] + "failure_taxonomy": [], + "blind_pair_count": 5 + }, + "output_execution": { + "ok": true, + "summary": { + "case_count": 5, + "variant_run_count": 10, + "command_executed_count": 10, + "model_executed_count": 0, + "recorded_fixture_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "token_estimated_count": 10, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "failure_count": 0, + "gate_pass": true + }, + "runner": { + "mode": "command", + "command": [ + "python3", + "scripts/local_output_eval_runner.py" + ], + "timeout_seconds": 30.0 + }, + "failures": [] + }, + "output_blind_review": { + "summary": { + "pair_count": 5, + "answer_key_separate": true, + "with_skill_hidden_count": 5 + }, + "seed": "yao-output-eval-blind-v1", + "pair_count": 5, + "answer_key_separate": true + }, + "output_review_adjudication": { + "ok": true, + "summary": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "agreement_count": 0, + "disagreement_count": 0, + "invalid_decision_count": 0, + "agreement_rate": null, + "needs_review": true, + "failure_count": 0 + }, + "reviewer": "", + "reviewed_at": "", + "failures": [] }, "runtime_conformance": { "target_count": 5, "pass_count": 5, "fail_count": 0 }, + "runtime_permissions": { + "ok": true, + "summary": { + "target_count": 3, + "pass_count": 3, + "fail_count": 0, + "native_enforcement_count": 0, + "metadata_fallback_count": 3, + "residual_risk_count": 3, + "required_capability_count": 3, + "failure_count": 0 + }, + "expected_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "targets": [ + { + "target": "openai", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "claude", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "generic", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + } + ], + "failures": [] + }, "trust_security": { - "scanned_files": 127, - "script_count": 50, + "scanned_files": 146, + "script_count": 62, + "internal_module_count": 3, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], "network_script_count": 2, + "network_policy_covered_count": 2, + "network_policy_missing_count": 0, + "file_write_script_count": 47, + "permission_required_count": 3, + "permission_approved_count": 3, + "permission_missing_count": 0, + "permission_invalid_count": 0, + "permission_expired_count": 0, + "help_smoke_checked_count": 59, + "help_smoke_failed_count": 0, "interactive_script_count": 0, - "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" + "package_hash_scope": "source-contract-without-generated-reports", + "package_hash_file_count": 146, + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" }, "skill_atlas": { "skill_count": 12, + "actionable_skill_count": 1, "route_collision_count": 5, + "actionable_route_collision_count": 0, "owner_gap_count": 9, + "actionable_owner_gap_count": 0, "stale_count": 10, + "actionable_stale_count": 0, "shared_resource_count": 0, - "no_route_opportunity_count": 0 + "no_route_opportunity_count": 0, + "non_actionable_issue_count": 24 + }, + "registry_distribution": { + "ok": true, + "package": { + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe", + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "agent-skills": "pass", + "vscode": "pass", + "generic": "pass", + "agent-skills-compatible": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "package_verification": "reports/package_verification.json", + "install_simulated": true, + "install_simulation": "reports/install_simulation.json" + }, + "generated_at": "2026-06-13" + }, + "failures": [], + "warnings": [] + }, + "package_verification": { + "ok": true, + "summary": { + "target_count": 3, + "adapter_count": 3, + "archive_present": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "archive_entry_count": 483, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "install_simulation": { + "ok": true, + "summary": { + "archive_present": true, + "archive_entry_count": 483, + "archive_extracted": true, + "entrypoint_loaded": true, + "manifest_loaded": true, + "interface_loaded": true, + "adapter_count": 3, + "install_root_is_temp": false, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "upgrade_check": { + "ok": true, + "summary": { + "declared_bump": "minor", + "recommended_bump": "minor", + "breaking_change_count": 0, + "added_target_count": 1, + "metadata_change_count": 1, + "checksum_change_count": 2, + "failure_count": 0, + "warning_count": 0 + }, + "upgrade_diff": { + "name_changed": false, + "added_targets": [ + "agent-skills-compatible" + ], + "removed_targets": [], + "compatibility_changes": [ + { + "target": "agent-skills", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "agent-skills-compatible", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "vscode", + "from": "missing", + "to": "pass", + "direction": "improved" + } + ], + "metadata_changes": [ + { + "field": "description", + "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + } + ], + "checksum_changes": [ + { + "field": "archive_sha256", + "from": "", + "to": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + { + "field": "package_sha256", + "from": "0000000000000000000000000000000000000000000000000000000000000000", + "to": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" + } + ] + }, + "release_notes": [ + "Recommended version bump: minor.", + "Added targets: agent-skills-compatible.", + "Package or archive checksum changed; reviewers should verify package artifacts before release.", + "Package verification evidence: reports/package_verification.md." + ], + "failures": [], + "warnings": [] + }, + "adoption_drift": { + "ok": true, + "summary": { + "event_count": 1, + "activation_count": 1, + "accepted_count": 1, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 100.0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "low", + "event_types": { + "skill_activation": 1 + }, + "failure_types": {} + }, + "next_iteration_candidates": [], + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "failures": [] + }, + "review_waivers": { + "ok": true, + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [] + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "failures": [], + "warnings": [] + }, + "review_annotations": { + "ok": true, + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [] }, "synthesis_highlights": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed.", "Borrow the discipline of defining what the skill should not own before growing the package." ], "artifact_design": { - "design_system": "review studio", + "design_system": "metric editorial", "highlights": [ "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", "Name the working directory and required inputs for commands.", @@ -872,24 +1372,24 @@ ] }, "prompt_quality": { - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "highlights": [ - "Primary prompt task family: Dialogue interaction.", - "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", - "Completeness: 80/100.", - "Clarity: 80/100." + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." ] }, "system_model": { "stability": { - "score": 86, - "band": "stable-first-pass" + "score": 100, + "band": "system-ready" }, "highlights": [ - "Stability: stable-first-pass (86/100).", - "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", - "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." ] }, "metadata": { @@ -905,82 +1405,56 @@ } }, "intent_confidence": { - "score": 30, - "band": "low", - "gate_passed": false, + "score": 100, + "band": "high", + "gate_passed": true, "strengths": [ - "The recurring job is concrete enough to anchor the package." + "The recurring job is concrete enough to anchor the package.", + "Real input shape is explicit.", + "The hand-back output is concrete.", + "Boundary exclusions are already explicit.", + "Operational constraints are visible." ], - "gaps": [ - { - "key": "real_inputs", - "label": "Real inputs are missing", - "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", - "severity": "high" - }, - { - "key": "primary_output", - "label": "Primary output is missing", - "reason": "The package does not yet know what it must hand back.", - "severity": "high" - }, - { - "key": "exclusions", - "label": "Near-neighbor exclusions are missing", - "reason": "The route may blur into nearby requests without an exclusion list.", - "severity": "high" - }, - { - "key": "constraints", - "label": "Constraints are missing", - "reason": "The package does not yet know which tradeoffs matter most.", - "severity": "high" - }, - { - "key": "standards", - "label": "Quality bar is implied, not explicit", - "reason": "The first evaluation target is still underspecified.", - "severity": "medium" - } - ], - "follow_up_questions": [ - { - "slot": "real_inputs", - "question": "What material will people actually hand to this skill in practice?", - "why": "Real input shape decides whether references, scripts, or examples are needed.", - "list": true, - "label": "Real inputs are missing", - "severity": "high" - }, - { - "slot": "primary_output", - "question": "What finished hand-back should this skill return so the next person can keep moving?", - "why": "The output is the anchor for package design and review.", - "list": false, - "label": "Primary output is missing", - "severity": "high" - }, - { - "slot": "exclusions", - "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?", - "why": "Exclusions are the fastest route to better trigger quality.", - "list": true, - "label": "Near-neighbor exclusions are missing", - "severity": "high" - } - ], - "anchor_sentence": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "gaps": [], + "follow_up_questions": [], + "anchor_sentence": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.", + "recommended_action": "Intent is clear enough to package the first routeable version.", "context": { - "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "real_inputs": [], - "primary_output": "", + "job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "real_inputs": [ + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" + ], + "primary_output": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "exclusions": [], - "constraints": [], - "standards": [], - "correction": "", - "user_references": [] + "exclusions": [ + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill", + "raw private material that was not intentionally supplied as skill evidence", + "platform-specific plugin creation when the request is not about an agent skill package" + ], + "constraints": [ + "keep SKILL.md lean and route primarily through frontmatter description", + "put durable guidance in references, executable logic in scripts, and evidence in reports", + "default to the lightest reliable mode before adding governance weight", + "preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets", + "avoid raw prompt, output, transcript, or private content in telemetry" + ], + "standards": [ + "trigger boundaries must be tested with should-trigger and should-not-trigger cases", + "production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence", + "governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation", + "generated reports should be bilingual or reviewer-friendly when they are user-facing", + "each new asset must earn its place by reducing ambiguity, risk, or repeated work" + ], + "correction": "The 2.0 direction reframes this package from a Meta Skill factory into a Skill OS with Skill IR, compiler contracts, output eval, runtime conformance, trust, registry, Review Studio, Skill Atlas, and telemetry loops.", + "user_references": [ + "Yao Meta Skill 2.0 upgrade plan", + "Kami-style white report layout requirements for generated skill reports", + "local Waza engineering habits for review, design, diagnosis, and implementation discipline" + ] } }, "intent_dialogue": { @@ -1665,7 +2139,8 @@ "delta": 100.0, "regression_count": 0, "gate_pass": true, - "failure_taxonomy": [] + "failure_taxonomy": [], + "blind_pair_count": 5 }, "results": [ { @@ -2454,9 +2929,3715 @@ } ], "failures": [], + "blind_review": { + "pack": "reports/output_blind_review_pack.json", + "answer_key": "reports/output_blind_answer_key.json", + "pair_count": 5 + }, "artifacts": { "json": "reports/output_quality_scorecard.json", - "markdown": "reports/output_quality_scorecard.md" + "markdown": "reports/output_quality_scorecard.md", + "blind_review_pack_json": "reports/output_blind_review_pack.json", + "blind_review_pack_md": "reports/output_blind_review_pack.md", + "blind_answer_key_json": "reports/output_blind_answer_key.json" + } + }, + "output_execution": { + "schema_version": "1.0", + "ok": true, + "cases": "evals/output/cases.jsonl", + "runner": { + "mode": "command", + "command": [ + "python3", + "scripts/local_output_eval_runner.py" + ], + "timeout_seconds": 30.0 + }, + "summary": { + "case_count": 5, + "variant_run_count": 10, + "command_executed_count": 10, + "model_executed_count": 0, + "recorded_fixture_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "token_estimated_count": 10, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "failure_count": 0, + "gate_pass": true + }, + "runs": [ + { + "case_id": "skill-package-contract", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.22, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 16, + "output_tokens": 17, + "total_tokens": 33, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "failed_assertions": [ + "has-entrypoint", + "has-interface", + "has-report", + "has-resource-boundary" + ], + "output_sha256": "ece26a6edbbbea86a93faaf9f0eadf88efbae12f1c3ad81da9376cf2a889d101", + "failure": "" + }, + { + "case_id": "skill-package-contract", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.12, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 16, + "output_tokens": 57, + "total_tokens": 73, + "estimated": true + }, + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "8bee0131e17fdbe87e48457130ac6aeaa55ea30c919a45e77109852609f33eba", + "failure": "" + }, + { + "case_id": "output-eval-expectation", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.12, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 24, + "output_tokens": 12, + "total_tokens": 36, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "failed_assertions": [ + "has-baseline-comparison", + "has-assertion-grading", + "has-scorecard", + "has-failure-taxonomy" + ], + "output_sha256": "98fa332fb493b5dc14656a9d65a6c0a9c25ff61a14b71b647aad89cbdbd3ab35", + "failure": "" + }, + { + "case_id": "output-eval-expectation", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 30.03, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 24, + "output_tokens": 56, + "total_tokens": 80, + "estimated": true + }, + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "8efdbfefa6a36603682b7063b1753f9f939b3810c082d29686e297906d9f51c8", + "failure": "" + }, + { + "case_id": "ir-before-packaging", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.11, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 18, + "output_tokens": 15, + "total_tokens": 33, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "failed_assertions": [ + "has-ir-first", + "has-core-ir-fields", + "has-targets", + "has-semantic-preservation" + ], + "output_sha256": "07a4aa9ee9e48341d4b84d5cb9b6414901aa2c8841340d67447e187240c6c4c2", + "failure": "" + }, + { + "case_id": "ir-before-packaging", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 26.68, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 18, + "output_tokens": 62, + "total_tokens": 80, + "estimated": true + }, + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "707fde419b3d61e921561b1c4a8523e1f7b53b1acd5758a47ce21d9ed711fde0", + "failure": "" + }, + { + "case_id": "near-neighbor-boundary", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.07, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 19, + "total_tokens": 36, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 3, + "failed_assertions": [ + "declines-skill", + "names-near-neighbor", + "requires-reuse-signal" + ], + "output_sha256": "648ebaa6e09486d05f33fa6639dc973b029d115158588fd192efbe15ccaaeac4", + "failure": "" + }, + { + "case_id": "near-neighbor-boundary", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 27.57, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 48, + "total_tokens": 65, + "estimated": true + }, + "score": 100.0, + "passed_count": 3, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "17a3a5286ad573dea86e51f82c04b596babaecb92cde8879f09e7c519205e2e1", + "failure": "" + }, + { + "case_id": "file-backed-governed-package", + "variant": "baseline", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 29.2, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 20, + "total_tokens": 37, + "estimated": true + }, + "score": 0.0, + "passed_count": 0, + "failed_count": 5, + "failed_assertions": [ + "uses-file-backed-evidence", + "has-governance", + "has-output-contract", + "has-trust-and-scorecard", + "does-not-invent-evidence" + ], + "output_sha256": "6e5827219459dee34143e4f11663415db80c3111291f75fb6c2115c7bda4cf30", + "failure": "" + }, + { + "case_id": "file-backed-governed-package", + "variant": "with_skill", + "status": "pass", + "execution_mode": "command", + "model_executed": false, + "command_executed": true, + "duration_ms": 26.94, + "provider": "local-output-eval-runner", + "model": "", + "usage": { + "input_tokens": 17, + "output_tokens": 81, + "total_tokens": 98, + "estimated": true + }, + "score": 100.0, + "passed_count": 5, + "failed_count": 0, + "failed_assertions": [], + "output_sha256": "bcfdf036666d406abbc945fc42b93eb74d7074118893e540253c7d8715baad73", + "failure": "" + } + ], + "failures": [], + "artifacts": { + "json": "reports/output_execution_runs.json", + "markdown": "reports/output_execution_runs.md" + } + }, + "output_blind_review": { + "schema_version": "1.0", + "seed": "yao-output-eval-blind-v1", + "summary": { + "pair_count": 5, + "answer_key_separate": true, + "with_skill_hidden_count": 5 + }, + "pairs": [ + { + "case_id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "metadata": {}, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0 + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0 + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0 + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "skill-package-contract:A", + "output": "Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/." + }, + "variant_b": { + "blind_id": "skill-package-contract:B", + "output": "I can write a prompt for that workflow and include a short checklist." + } + }, + { + "case_id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "metadata": {}, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0 + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0 + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0 + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "output-eval-expectation:A", + "output": "Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md." + }, + "variant_b": { + "blind_id": "output-eval-expectation:B", + "output": "Add more examples and run the trigger eval again." + } + }, + { + "case_id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "metadata": {}, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0 + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0 + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0 + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "ir-before-packaging:A", + "output": "Export folders for each platform and adjust files as needed." + }, + "variant_b": { + "blind_id": "ir-before-packaging:B", + "output": "Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract." + } + }, + { + "case_id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "metadata": { + "case_type": "near_neighbor", + "tier": "production" + }, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0 + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0 + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "near-neighbor-boundary:A", + "output": "Create a SKILL.md and a reusable workflow anyway so future notes can use it." + }, + "variant_b": { + "blind_id": "near-neighbor-boundary:B", + "output": "Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract." + } + }, + { + "case_id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "metadata": { + "case_type": "boundary", + "tier": "governed" + }, + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0 + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0 + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0 + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0 + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0 + } + ], + "variant_a": { + "blind_id": "file-backed-governed-package:A", + "output": "Draft a release announcement with the changelog, support notes, and owner name." + }, + "variant_b": { + "blind_id": "file-backed-governed-package:B", + "output": "Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them." + } + } + ] + }, + "output_review_adjudication": { + "schema_version": "1.0", + "ok": true, + "summary": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "agreement_count": 0, + "disagreement_count": 0, + "invalid_decision_count": 0, + "agreement_rate": null, + "needs_review": true, + "failure_count": 0 + }, + "reviewer": "", + "reviewed_at": "", + "artifacts": { + "blind_pack": "reports/output_blind_review_pack.json", + "answer_key": "reports/output_blind_answer_key.json", + "decisions": "reports/output_review_decisions.json", + "json": "reports/output_review_adjudication.json", + "markdown": "reports/output_review_adjudication.md" + }, + "template_written": false, + "pairs": [ + { + "case_id": "skill-package-contract", + "status": "pending", + "expected_winner_variant": "A", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Turn this repeated workflow into a reusable team skill package." + }, + { + "case_id": "output-eval-expectation", + "status": "pending", + "expected_winner_variant": "A", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline." + }, + { + "case_id": "ir-before-packaging", + "status": "pending", + "expected_winner_variant": "B", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets." + }, + { + "case_id": "near-neighbor-boundary", + "status": "pending", + "expected_winner_variant": "B", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "I only need a one-off summary of these notes, not a reusable process." + }, + { + "case_id": "file-backed-governed-package", + "status": "pending", + "expected_winner_variant": "B", + "reviewer_winner_variant": "", + "confidence": null, + "reason": "", + "prompt": "Turn the attached release brief source into a governed skill package." + } + ], + "failures": [] + }, + "compiled_targets": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-13", + "skill_dir": ".", + "summary": { + "target_count": 4, + "pass_count": 4, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "schema_version": "1.0", + "target": "openai", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "openai", + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "degradation_strategy": "metadata-adapter", + "target_runtime": { + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "openai", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "openai", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation": { + "policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ] + }, + "scripts": { + "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/openai/agents/openai.yaml", + "targets/openai/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + "client-native script permission prompts are represented as permission contract metadata" + ] + }, + "install_scope": "plugin or skill package consumer" + }, + "target_transform": { + "target": "openai", + "adapter_mode": "metadata-adapter", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "metadata-adapter", + "permission_representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "permission_enforcement": "metadata-only" + }, + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "claude", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "claude", + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "degradation_strategy": "neutral-source-plus-adapter", + "target_runtime": { + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Claude-compatible neutral source folder with adapter notes" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "claude", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "claude", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation": { + "policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ] + }, + "scripts": { + "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "metadata-fallback", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/claude/README.md", + "targets/claude/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + "vendor-native metadata fields are carried as adapter JSON and README notes" + ] + }, + "install_scope": "user or project skill directory" + }, + "target_transform": { + "target": "claude", + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source-plus-adapter", + "permission_representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "permission_enforcement": "metadata-fallback" + }, + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "generic", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "generic", + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills compatible neutral package" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "generic", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "representation": "targets/generic/adapter.json::target_permission_contract", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "generic", + "native_surface": "Agent Skills compatible neutral package", + "activation": { + "policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "targets/generic/adapter.json" + ] + }, + "scripts": { + "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "targets/generic/adapter.json", + "reports/review-studio.html" + ], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [] + }, + "install_scope": "generic Agent Skills compatible root" + }, + "target_transform": { + "target": "generic", + "adapter_mode": "agent-skills-compatible", + "generated_files": [ + "targets/generic/adapter.json" + ], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy" + }, + "preserved_semantics": [ + "trigger", + "workflow-counts", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "targets/generic/adapter.json::target_permission_contract", + "native_surface": "Agent Skills compatible neutral package", + "activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + }, + { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "status": "pass", + "compiler": { + "name": "yao-skill-ir-compiler", + "schema_version": "1.0", + "source": "skill-ir", + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" + }, + "source": { + "skill_dir": ".", + "canonical_metadata": "agents/interface.yaml", + "declared_targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "compiled_contract": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/distribution-registry-method.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/review-waiver-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", + "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/validate_skill.py", + "scripts/verify_package.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ], + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + } + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill", + "counts": { + "trigger": 26, + "output": 3, + "adversarial": 1, + "baseline": 1 + } + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "runtime": { + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "canonical_format": "agent-skills" + }, + "permissions": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "source_files_count": 6, + "target": "agent-skills-compatible", + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "degradation_strategy": "neutral-source", + "target_runtime": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "native_surface": "Agent Skills standard source tree" + } + }, + "permission_contract": { + "schema_version": "1.0", + "source": "reports/security_trust_report.json", + "source_available": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true, + "capabilities": { + "network": { + "required": true, + "script_count": 2, + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "review_reason": "Outbound hosts must match security/network_policy.json." + }, + "file_write": { + "required": true, + "script_count": 47, + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "review_reason": "Scripts write local files or generated artifacts." + }, + "subprocess": { + "required": true, + "script_count": 7, + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "review_reason": "Scripts spawn local commands and need operator review." + }, + "interactive": { + "required": false, + "script_count": 0, + "scripts": [], + "review_reason": "" + } + }, + "network_policy": { + "source": "security/network_policy.json", + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [] + }, + "help_smoke": { + "enabled": true, + "checked_count": 59, + "failed_count": 0, + "failed_scripts": [] + }, + "trust_summary": { + "secret_findings": 0, + "network_script_count": 2, + "file_write_script_count": 47, + "subprocess_script_count": 7, + "interactive_script_count": 0, + "help_smoke_failed_count": 0 + } + }, + "target_permission_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "permission_model": "metadata-only", + "native_enforcement": false, + "representation": "adapter metadata", + "review_required": true, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "capability_counts": { + "network": 2, + "file_write": 47, + "subprocess": 7, + "interactive": 0 + }, + "evidence": "reports/security_trust_report.json", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility." + }, + "target_native_contract": { + "schema_version": "1.0", + "target": "agent-skills-compatible", + "native_surface": "Agent Skills standard source tree", + "activation": { + "policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "trigger_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "manual_activation_supported": true, + "automatic_activation_note": "Depends on the target client route/catalog implementation." + }, + "resources": { + "strategy": "Keep optional directories as relative resources next to SKILL.md.", + "counts": { + "references": 32, + "scripts": 62, + "assets": 2, + "reports": 41 + }, + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ] + }, + "scripts": { + "strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "script_count": 62, + "help_smoke_failed_count": 0 + }, + "permissions": { + "enforcement": "consumer-enforced-or-metadata-only", + "native_enforcement": false, + "declared_capabilities": [ + "network", + "file_write", + "subprocess" + ], + "review_required": true + }, + "review": { + "artifacts": [ + "SKILL.md", + "agents/interface.yaml", + "reports/review-studio.html" + ], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [] + }, + "install_scope": "Agent Skills source root" + }, + "target_transform": { + "target": "agent-skills-compatible", + "adapter_mode": "neutral-agent-skills-source", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json" + }, + "preserved_semantics": [ + "trigger", + "workflow", + "resources", + "eval-plan", + "risk", + "governance", + "runtime", + "trust", + "permissions" + ], + "degradation_strategy": "neutral-source", + "permission_representation": "adapter metadata", + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only" + }, + "unsupported_features": [], + "warnings": [], + "failures": [] + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "json": "reports/compiled_targets.json", + "markdown": "reports/compiled_targets.md" } }, "conformance": { @@ -2493,6 +6674,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -2501,8 +6683,7 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [] @@ -2537,6 +6718,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -2545,8 +6727,7 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [] @@ -2581,6 +6762,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -2589,12 +6771,11 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [ - "agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + "agent-skills uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0." ] }, { @@ -2627,6 +6808,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -2635,12 +6817,11 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [ - "vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + "vscode uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0." ] }, { @@ -2673,6 +6854,7 @@ "Skill IR description matches frontmatter", "references resource resolves: references/artifact-design-doctrine.md", "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/distribution-registry-method.md", "references resource resolves: references/eval-playbook.md", "references resource resolves: references/gate-selection.md", "references resource resolves: references/governance.md", @@ -2681,8 +6863,7 @@ "references resource resolves: references/iteration-philosophy.md", "references resource resolves: references/non-skill-decision-tree.md", "references resource resolves: references/operating-modes.md", - "references resource resolves: references/output-eval-method.md", - "references resource resolves: references/output-quality-risk.md" + "references resource resolves: references/output-eval-method.md" ], "failures": [], "warnings": [] @@ -2698,426 +6879,1170 @@ "markdown": "reports/conformance_matrix.md" } }, + "runtime_permissions": { + "schema_version": "1.0", + "ok": true, + "skill_dir": ".", + "package_dir": "tests/tmp_review_studio/dist", + "expected_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "summary": { + "target_count": 3, + "pass_count": 3, + "fail_count": 0, + "native_enforcement_count": 0, + "metadata_fallback_count": 3, + "residual_risk_count": 3, + "required_capability_count": 3, + "failure_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", + "permission_model": "metadata-only", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "openai adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "openai adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "openai adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "openai permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "openai target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "openai target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "openai target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "openai target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "openai target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "openai review_required matches whether capabilities are required" + }, + { + "key": "openai-yaml-present", + "passed": true, + "detail": "OpenAI permission metadata YAML is readable" + }, + { + "key": "openai-yaml-permissions", + "passed": true, + "detail": "OpenAI YAML permission contract mirrors expected capabilities" + }, + { + "key": "openai-yaml-native-flag", + "passed": true, + "detail": "OpenAI YAML declares native_enforcement as a boolean" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "claude", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "claude adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "claude adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "claude adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "claude permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "claude target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "claude target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "claude target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "claude target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "claude target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "claude review_required matches whether capabilities are required" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "generic", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "generic adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "generic adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "generic adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "generic permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "generic target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "generic target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "generic target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "generic target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "generic target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "generic review_required matches whether capabilities are required" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + } + ], + "failures": [], + "artifacts": { + "json": "reports/runtime_permission_probes.json", + "markdown": "reports/runtime_permission_probes.md" + } + }, "trust": { "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 127, - "script_count": 50, + "scanned_files": 146, + "script_count": 62, + "internal_module_count": 3, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], "network_script_count": 2, + "network_policy_covered_count": 2, + "network_policy_missing_count": 0, + "file_write_script_count": 47, + "permission_required_count": 3, + "permission_approved_count": 3, + "permission_missing_count": 0, + "permission_invalid_count": 0, + "permission_expired_count": 0, + "help_smoke_checked_count": 59, + "help_smoke_failed_count": 0, "interactive_script_count": 0, - "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" + "package_hash_scope": "source-contract-without-generated-reports", + "package_hash_file_count": 146, + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" }, "failures": [], - "warnings": [ - "Scripts without argparse/help surface: scripts/render_context_reports.py, scripts/render_social_preview.py, scripts/skill_report_charts.py, scripts/skill_report_metrics.py, scripts/skill_report_model.py", - "Network-capable scripts require bounded host policy: scripts/check_update.py, scripts/github_benchmark_scan.py" - ], + "warnings": [], "secrets": [], "scripts": [ { - "path": "scripts/build_confusion_matrix.py", + "path": "scripts/adjudicate_output_review.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/build_confusion_matrix.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/build_skill_atlas.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/check_update.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": true, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [ + "https://raw.githubusercontent.com/yaojingang/yao-meta-skill/main/VERSION", + "https://raw.githubusercontent.com/yaojingang/yao-meta-skill/main/manifest.json" + ], + "network_hosts": [ + "raw.githubusercontent.com" + ] }, { "path": "scripts/ci_test.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/collect_feedback.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/compile_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/context_sizer.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/create_iteration_snapshot.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/cross_packager.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/diff_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/export_skill_ir.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/github_benchmark_scan.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": true, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [ + "https://api.github.com" + ], + "network_hosts": [ + "api.github.com" + ] }, { "path": "scripts/governance_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/init_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/judge_blind_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/lint_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/local_output_eval_runner.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/optimize_description.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/probe_runtime_permissions.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/promotion_checker.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/registry_audit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_adoption_drift_report.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_artifact_design_profile.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_baseline_compare.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_context_reports.py", - "has_argparse": false, + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_description_drift_history.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_eval_dashboard.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_intent_confidence.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_intent_dialogue.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_iteration_directions.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_iteration_ledger.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_output_risk_profile.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_portability_report.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_prompt_quality_profile.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_reference_scan.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_reference_synthesis.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_regression_history.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_review_annotations.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_review_studio.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_review_viewer.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_review_waivers.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_skill_overview.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_social_preview.py", - "has_argparse": false, + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [ + "http://www.w3.org/2000/svg" + ], + "network_hosts": [ + "www.w3.org" + ] }, { "path": "scripts/render_system_model.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/resource_boundary_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_conformance_suite.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_description_optimization_suite.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_eval_suite.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_output_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/run_output_execution.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/simulate_install.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/skill_report_charts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_overview.py to render inline SVG report charts.", "has_argparse": false, "has_main_guard": false, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/skill_report_metrics.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to calculate overview report metrics.", "has_argparse": false, "has_main_guard": false, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/skill_report_model.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_overview.py to build the v2 report data model.", "has_argparse": false, "has_main_guard": false, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/sync_local_install.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/trigger_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/trust_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/upgrade_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/validate_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/verify_package.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/yao.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] } ], "dependencies": { @@ -3129,6 +8054,782 @@ ], "unpinned": [] }, + "network_policy": { + "present": true, + "path": "security/network_policy.json", + "schema_version": "1.0", + "network_script_count": 2, + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [], + "default_policy": { + "require_https": true, + "custom_hosts": "deny-by-default", + "review_rule": "Network-capable scripts must declare allowed_hosts and justify any explicit custom-host override." + } + }, + "help_smoke": { + "enabled": true, + "timeout_seconds": 5.0, + "candidate_count": 59, + "checked_count": 59, + "passed_count": 59, + "failed_count": 0, + "skipped_count": 3, + "failed_scripts": [], + "results": [ + { + "path": "scripts/adjudicate_output_review.py", + "command": "python3 scripts/adjudicate_output_review.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: adjudicate_output_review.py [-h] [--blind-pack BLIND_PACK]\n [--answer-key ANSWER_KEY]\n [--decisions DECISIONS]\n [--output-json OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/build_confusion_matrix.py", + "command": "python3 scripts/build_confusion_matrix.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: build_confusion_matrix.py [-h] [--cases CASES]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--history-snapshot-output HISTORY_", + "stderr_excerpt": "" + }, + { + "path": "scripts/build_skill_atlas.py", + "command": "python3 scripts/build_skill_atlas.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: build_skill_atlas.py [-h] [--workspace-root WORKSPACE_ROOT]\n [--output-dir OUTPUT_DIR]\n [--report-html REPORT_HTML]\n [--report-json REPORT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/check_update.py", + "command": "python3 scripts/check_update.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: check_update.py [-h] [--root ROOT] [--cache-path CACHE_PATH]\n [--version-url VERSION_URL]\n [--manifest-url MANIFEST_URL] [--timeout TIMEOUT]\n [--max-age-days MAX_AGE_D", + "stderr_excerpt": "" + }, + { + "path": "scripts/ci_test.py", + "command": "python3 scripts/ci_test.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: ci_test.py [-h] [--tail-lines TAIL_LINES] [targets ...]\n\nRun CI test targets with compact logging.\n\npositional arguments:\n targets Optional target override.\n\noptions:\n -h, --help show this help message and ", + "stderr_excerpt": "" + }, + { + "path": "scripts/collect_feedback.py", + "command": "python3 scripts/collect_feedback.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: collect_feedback.py [-h] [--note NOTE] [--rating RATING]\n [--category CATEGORY]\n [--recommended-action RECOMMENDED_ACTION]\n [--output-json OUTPUT_JSON] [--", + "stderr_excerpt": "" + }, + { + "path": "scripts/compile_skill.py", + "command": "python3 scripts/compile_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: compile_skill.py [-h] [--target TARGET] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD] [--generated-at GENERATED_AT]\n [skill_dir]\n\nCompile Skill IR into target-specific semantic con", + "stderr_excerpt": "" + }, + { + "path": "scripts/context_sizer.py", + "command": "python3 scripts/context_sizer.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: context_sizer.py [-h] [--json] skill_dir\n\nEstimate context size for a skill package.\n\npositional arguments:\n skill_dir Path to the skill directory\n\noptions:\n -h, --help show this help message and exit\n --json Emit machine-", + "stderr_excerpt": "" + }, + { + "path": "scripts/create_iteration_snapshot.py", + "command": "python3 scripts/create_iteration_snapshot.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: create_iteration_snapshot.py [-h] --target TARGET [--label LABEL]\n [--promotion-decisions PROMOTION_DECISIONS]\n [--route-scorecard ROUTE_SCORECARD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/cross_packager.py", + "command": "python3 scripts/cross_packager.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: cross_packager.py [-h] [--platform PLATFORM] [--output-dir OUTPUT_DIR]\n [--expectations EXPECTATIONS] [--zip]\n skill_dir\n\nGenerate lightweight cross-platform packaging artifacts.\n\nposit", + "stderr_excerpt": "" + }, + { + "path": "scripts/diff_eval.py", + "command": "python3 scripts/diff_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: diff_eval.py [-h] before after\n\nDiff two eval reports.\n\npositional arguments:\n before\n after\n\noptions:\n -h, --help show this help message and exit", + "stderr_excerpt": "" + }, + { + "path": "scripts/export_skill_ir.py", + "command": "python3 scripts/export_skill_ir.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: export_skill_ir.py [-h] [--output-json OUTPUT_JSON] [--validate-only]\n [skill_dir]\n\nExport a platform-neutral Skill IR document from a skill package.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help ", + "stderr_excerpt": "" + }, + { + "path": "scripts/github_benchmark_scan.py", + "command": "python3 scripts/github_benchmark_scan.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: github_benchmark_scan.py [-h] --query QUERY [--top-n TOP_N]\n [--fixture-dir FIXTURE_DIR]\n [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSO", + "stderr_excerpt": "" + }, + { + "path": "scripts/governance_check.py", + "command": "python3 scripts/governance_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: governance_check.py [-h] [--require-manifest] skill_dir\n\nCheck skill governance metadata and lifecycle readiness.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit\n --require-manifest", + "stderr_excerpt": "" + }, + { + "path": "scripts/init_skill.py", + "command": "python3 scripts/init_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: init_skill.py [-h] [--description DESCRIPTION] [--title TITLE]\n [--output-dir OUTPUT_DIR]\n [--mode {governed,library,production,scaffold}]\n [--archetype {governed,library,pr", + "stderr_excerpt": "" + }, + { + "path": "scripts/judge_blind_eval.py", + "command": "python3 scripts/judge_blind_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: judge_blind_eval.py [-h] [--description DESCRIPTION]\n [--description-file DESCRIPTION_FILE] --cases CASES\n [--semantic-config SEMANTIC_CONFIG]\n\nRun a rubric-based blind trigger judg", + "stderr_excerpt": "" + }, + { + "path": "scripts/lint_skill.py", + "command": "python3 scripts/lint_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: lint_skill.py [-h] skill_dir\n\nLint a skill package for basic structure issues.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit", + "stderr_excerpt": "" + }, + { + "path": "scripts/local_output_eval_runner.py", + "command": "python3 scripts/local_output_eval_runner.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: local_output_eval_runner.py [-h] [--execution-kind {command,model}]\n [--provider PROVIDER] [--model MODEL]\n\nLocal deterministic output-eval runner. It executes the runner contract\nwithout claiming pr", + "stderr_excerpt": "" + }, + { + "path": "scripts/optimize_description.py", + "command": "python3 scripts/optimize_description.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: optimize_description.py [-h] --description-file DESCRIPTION_FILE\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n --dev-cases DEV_CASES\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/probe_runtime_permissions.py", + "command": "python3 scripts/probe_runtime_permissions.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: probe_runtime_permissions.py [-h] [--package-dir PACKAGE_DIR]\n [--target {openai,claude,generic}]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/promotion_checker.py", + "command": "python3 scripts/promotion_checker.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: promotion_checker.py [-h] [--optimization-suite OPTIMIZATION_SUITE]\n [--route-scorecard ROUTE_SCORECARD]\n [--output-json OUTPUT_JSON]\n [--output-md OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/registry_audit.py", + "command": "python3 scripts/registry_audit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: registry_audit.py [-h] [--registry-dir REGISTRY_DIR]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [skill_dir]\n\nBuild and ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_adoption_drift_report.py", + "command": "python3 scripts/render_adoption_drift_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_adoption_drift_report.py [-h] [--events-jsonl EVENTS_JSONL]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_artifact_design_profile.py", + "command": "python3 scripts/render_artifact_design_profile.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_artifact_design_profile.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender artifact design direction and visual qua", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_baseline_compare.py", + "command": "python3 scripts/render_baseline_compare.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_baseline_compare.py [-h] --entry ENTRY\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a baseline comparison report from description optimization ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_context_reports.py", + "command": "python3 scripts/render_context_reports.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_context_reports.py [-h] [--generated-at GENERATED_AT]\n\nRender context budget reports for root and example skills.\n\noptions:\n -h, --help show this help message and exit\n --generated-at GENERATED_AT\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_description_drift_history.py", + "command": "python3 scripts/render_description_drift_history.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_description_drift_history.py [-h] [--history-dir HISTORY_DIR]\n [--output OUTPUT]\n\nRender description optimization drift history.\n\noptions:\n -h, --help show this help messag", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_eval_dashboard.py", + "command": "python3 scripts/render_eval_dashboard.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_eval_dashboard.py [-h] [--eval-dir EVAL_DIR]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_intent_confidence.py", + "command": "python3 scripts/render_intent_confidence.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_intent_confidence.py [-h] [--context-json CONTEXT_JSON]\n [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\n", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_intent_dialogue.py", + "command": "python3 scripts/render_intent_dialogue.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_intent_dialogue.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender an intent dialogue guide for a skill package.\n\npositional argume", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_iteration_directions.py", + "command": "python3 scripts/render_iteration_directions.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_iteration_directions.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender the top three next iteration directions for a ski", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_iteration_ledger.py", + "command": "python3 scripts/render_iteration_ledger.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_iteration_ledger.py [-h] [--history-dir HISTORY_DIR]\n [--description-history-dir DESCRIPTION_HISTORY_DIR]\n [--route-scorecard ROUTE_SCORECARD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_output_risk_profile.py", + "command": "python3 scripts/render_output_risk_profile.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_output_risk_profile.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender predicted output failure modes and quality constrain", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_portability_report.py", + "command": "python3 scripts/render_portability_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_portability_report.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a portability score from neutral metadata, contracts, and snapshots.\n\noptions:\n -h, --help ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_prompt_quality_profile.py", + "command": "python3 scripts/render_prompt_quality_profile.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_prompt_quality_profile.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender prompt behavior quality, RTF mapping, and n", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_reference_scan.py", + "command": "python3 scripts/render_reference_scan.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_reference_scan.py [-h] [--external-reference EXTERNAL_REFERENCE]\n [--user-reference USER_REFERENCE]\n [--local-constraint LOCAL_CONSTRAINT]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_reference_synthesis.py", + "command": "python3 scripts/render_reference_synthesis.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_reference_synthesis.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a multi-source reference synthesis report for a skil", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_regression_history.py", + "command": "python3 scripts/render_regression_history.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_regression_history.py [-h] [--history-dir HISTORY_DIR]\n [--output OUTPUT]\n\nRender regression history from milestone snapshots.\n\noptions:\n -h, --help show this help message and exi", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_annotations.py", + "command": "python3 scripts/render_review_annotations.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_annotations.py [-h] [--annotations-json ANNOTATIONS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD] [--write-template]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_studio.py", + "command": "python3 scripts/render_review_studio.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_studio.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender Review Studio 2.0 for a skill package.\n\npositional arguments:\n sk", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_viewer.py", + "command": "python3 scripts/render_review_viewer.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_viewer.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a compact HTML review viewer for a skill package.\n\npositional argu", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_waivers.py", + "command": "python3 scripts/render_review_waivers.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_waivers.py [-h] [--waivers-json WAIVERS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_skill_overview.py", + "command": "python3 scripts/render_skill_overview.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_overview.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender the HTML skill report for a skill package.\n\npositional argument", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_social_preview.py", + "command": "python3 scripts/render_social_preview.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_social_preview.py [-h] [--format {all,png,svg}]\n\nRender the repository social preview image assets.\n\noptions:\n -h, --help show this help message and exit\n --format {all,png,svg}\n Asset forma", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_system_model.py", + "command": "python3 scripts/render_system_model.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_system_model.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a systems-thinking model for a skill package.\n\npositional arguments:\n sk", + "stderr_excerpt": "" + }, + { + "path": "scripts/resource_boundary_check.py", + "command": "python3 scripts/resource_boundary_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: resource_boundary_check.py [-h]\n [--max-initial-tokens MAX_INITIAL_TOKENS]\n [--warn-skill-body-tokens WARN_SKILL_BODY_TOKENS]\n skill_d", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_conformance_suite.py", + "command": "python3 scripts/run_conformance_suite.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_conformance_suite.py [-h]\n [--target {openai,claude,agent-skills,vscode,generic}]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_description_optimization_suite.py", + "command": "python3 scripts/run_description_optimization_suite.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_description_optimization_suite.py [-h]\n [--history-snapshot-output HISTORY_SNAPSHOT_OUTPUT]\n [--snapshot-date SNAPSHOT_DATE]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_eval_suite.py", + "command": "python3 scripts/run_eval_suite.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_eval_suite.py [-h] [--eval-dir EVAL_DIR]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n [--output-file OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_output_eval.py", + "command": "python3 scripts/run_output_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_output_eval.py [-h] [--cases CASES] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--blind-pack-json BLIND_PACK_JSON]\n [--blind-pack-md BLIND_PACK", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_output_execution.py", + "command": "python3 scripts/run_output_execution.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_output_execution.py [-h] [--cases CASES]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--runner-command RUNNER_COMMAND]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/simulate_install.py", + "command": "python3 scripts/simulate_install.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: simulate_install.py [-h] [--package-dir PACKAGE_DIR]\n [--install-root INSTALL_ROOT]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--generated-at G", + "stderr_excerpt": "" + }, + { + "path": "scripts/sync_local_install.py", + "command": "python3 scripts/sync_local_install.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: sync_local_install.py [-h] [--root ROOT] [--install-dir INSTALL_DIR]\n [--dry-run]\n\nSync the current yao-meta-skill source into a managed local skill mirror.\n\noptions:\n -h, --help show this help", + "stderr_excerpt": "" + }, + { + "path": "scripts/trigger_eval.py", + "command": "python3 scripts/trigger_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: trigger_eval.py [-h] [--description DESCRIPTION]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description BASELINE_DESCRIPTION]\n [--baseline-description-file BA", + "stderr_excerpt": "" + }, + { + "path": "scripts/trust_check.py", + "command": "python3 scripts/trust_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: trust_check.py [-h] [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--skip-help-smoke]\n [--help-smoke-timeout HELP_SMOKE_TIMEOUT]\n [skill_dir]\n\nRun Skill OS trust a", + "stderr_excerpt": "" + }, + { + "path": "scripts/upgrade_check.py", + "command": "python3 scripts/upgrade_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: upgrade_check.py [-h] --previous-package-json PREVIOUS_PACKAGE_JSON\n [--current-package-json CURRENT_PACKAGE_JSON]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/validate_skill.py", + "command": "python3 scripts/validate_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: validate_skill.py [-h] skill_dir\n\nValidate a skill package.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit", + "stderr_excerpt": "" + }, + { + "path": "scripts/verify_package.py", + "command": "python3 scripts/verify_package.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: verify_package.py [-h] [--package-dir PACKAGE_DIR]\n [--expectations EXPECTATIONS]\n [--registry-json REGISTRY_JSON]\n [--output-json OUTPUT_JSON] [--output-md OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/yao.py", + "command": "python3 scripts/yao.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: yao.py [-h]\n {init,quickstart,validate,optimize-description,promote-check,review,release-snapshot,workspace-flow,report,skill-report,review-viewer,review-studio,reference-scan,github-benchmark-scan,intent-confidence,inte", + "stderr_excerpt": "" + } + ], + "skipped": [ + { + "path": "scripts/skill_report_charts.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_metrics.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_model.py", + "reason": "internal module" + } + ] + }, + "permission_governance": { + "present": true, + "path": "security/permission_policy.json", + "schema_version": "1.0", + "reviewed_at": "2026-06-13", + "required_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "approved_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "invalid_capabilities": [], + "expired_capabilities": [], + "approval_count": 3, + "required_count": 3, + "missing_count": 0, + "invalid_count": 0, + "expired_count": 0, + "approvals": [ + { + "capability": "network", + "status": "approved", + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "reviewer": "Yao Team", + "scope": "Only scripts declared in security/network_policy.json may perform outbound requests.", + "reason": "Network use is bounded to public GitHub metadata and benchmark discovery, with allowed hosts reviewed in security/network_policy.json.", + "expires_at": "2026-09-30", + "evidence": [ + "security/network_policy.json", + "reports/security_trust_report.md" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + }, + "validation": [] + }, + { + "capability": "file_write", + "status": "approved", + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "reviewer": "Yao Team", + "scope": "Local generated reports, registry metadata, temporary test outputs, and package artifacts under the skill workspace.", + "reason": "File writes are required to create auditable reports, compiled target contracts, package artifacts, and local test fixtures; package safety checks exclude generated temp outputs from archives.", + "expires_at": "2026-09-30", + "evidence": [ + "reports/package_verification.md", + "reports/install_simulation.md", + "security/permission_policy.md" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + }, + "validation": [] + }, + { + "capability": "subprocess", + "status": "approved", + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "reviewer": "Yao Team", + "scope": "Local deterministic validation, packaging, install simulation, and CI orchestration commands.", + "reason": "Subprocess use is limited to local verification and packaging flows that are surfaced through CLI help smoke tests and CI targets.", + "expires_at": "2026-09-30", + "evidence": [ + "reports/security_trust_report.md", + "reports/review-studio.html" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + }, + "validation": [] + } + ] + }, "trust_metadata": { "source_tier": "local", "remote_inline_execution": "forbid", @@ -3146,14 +8847,14 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 734, - "other_text_tokens": 606468, - "estimated_initial_load_tokens": 910, - "estimated_total_text_tokens": 607202, - "relevant_file_count": 241, + "skill_body_tokens": 811, + "other_text_tokens": 864534, + "estimated_initial_load_tokens": 987, + "estimated_total_text_tokens": 865345, + "relevant_file_count": 316, "unused_resource_dirs": [], "quality_signal_points": 130, - "quality_density": 142.9 + "quality_density": 131.7 } }, "promotion": { @@ -3953,11 +9654,41 @@ "workspace_root": ".", "summary": { "skill_count": 12, + "actionable_skill_count": 1, "route_collision_count": 5, + "actionable_route_collision_count": 0, "owner_gap_count": 9, + "actionable_owner_gap_count": 0, "stale_count": 10, + "actionable_stale_count": 0, "shared_resource_count": 0, - "no_route_opportunity_count": 0 + "no_route_opportunity_count": 0, + "non_actionable_issue_count": 24 + }, + "scope_policy": { + "present": true, + "path": "skill_atlas/policy.json", + "schema_version": "1.0", + "rules": [ + { + "path_prefix": "examples/", + "scope": "example", + "actionable": false, + "reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + }, + { + "path_prefix": "tests/fixtures/", + "scope": "test-fixture", + "actionable": false, + "reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + }, + { + "path_prefix": "geo-ranking-article-generator", + "scope": "embedded-generated-skill", + "actionable": false, + "reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + } + ] }, "catalog": { "workspace_root": ".", @@ -3980,11 +9711,13 @@ "agent-skills-compatible" ], "resources": [ + "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", + "scripts/compile_skill.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", @@ -3995,8 +9728,12 @@ "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", "scripts/render_context_reports.py", @@ -4012,8 +9749,10 @@ "scripts/render_reference_scan.py", "scripts/render_reference_synthesis.py", "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", "scripts/render_review_studio.py", "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", @@ -4022,16 +9761,21 @@ "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", "scripts/skill_report_charts.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/upgrade_check.py", "scripts/validate_skill.py", + "scripts/verify_package.py", "scripts/yao.py", "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", "references/governance.md", @@ -4052,12 +9796,14 @@ "references/regression-cause-taxonomy.md", "references/resource-boundaries.md", "references/review-studio-method.md", + "references/review-waiver-method.md", "references/runtime-conformance-method.md", "references/skill-archetypes.md", "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", "references/trust-security-method.md", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" @@ -4081,7 +9827,10 @@ "team", "transcripts", "workflows" - ] + ], + "atlas_scope": "release", + "actionable": true, + "scope_reason": "default release-actionable skill" }, { "name": "release-orchestrator", @@ -4136,7 +9885,10 @@ "stakeholder", "theory", "workflow" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -4172,7 +9924,10 @@ "states", "ui", "ux" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -4192,7 +9947,10 @@ "frontend", "merge", "review" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -4227,7 +9985,10 @@ "security", "states", "ui" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "incident-command-governor", @@ -4265,7 +10026,10 @@ "run", "severity", "standardize" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "note-cleanup", @@ -4299,7 +10063,10 @@ "separate", "structured", "summaries" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -4336,7 +10103,10 @@ "ui", "unsafe", "ux" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "geo-ranking-article-generator", @@ -4393,7 +10163,10 @@ "write", "writing", "生成式引擎优化" - ] + ], + "atlas_scope": "embedded-generated-skill", + "actionable": false, + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." }, { "name": "invalid-governance-skill", @@ -4412,7 +10185,10 @@ "governance", "test", "validation" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." }, { "name": "broken-yaml-skill", @@ -4433,7 +10209,10 @@ "packager", "testing", "yaml" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." }, { "name": "broken-skill", @@ -4453,16 +10232,24 @@ "fixture", "packager", "testing" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." } ], "summary": { "skill_count": 12, + "actionable_skill_count": 1, "route_collision_count": 5, + "actionable_route_collision_count": 0, "owner_gap_count": 9, + "actionable_owner_gap_count": 0, "stale_count": 10, + "actionable_stale_count": 0, "shared_resource_count": 0, - "no_route_opportunity_count": 0 + "no_route_opportunity_count": 0, + "non_actionable_issue_count": 24 } }, "route_collisions": [ @@ -4472,7 +10259,10 @@ "path_a": "examples/evolution-frontend-review/final", "path_b": "examples/evolution-frontend-review/v1", "score": 0.708, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "example", + "scope_b": "example" }, { "skill_a": "frontend-review", @@ -4480,7 +10270,10 @@ "path_a": "examples/evolution-frontend-review/final", "path_b": "examples/team-frontend-review/generated-skill", "score": 0.64, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "example", + "scope_b": "example" }, { "skill_a": "frontend-review", @@ -4488,7 +10281,10 @@ "path_a": "examples/evolution-frontend-review/v1", "path_b": "examples/team-frontend-review/generated-skill", "score": 0.538, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "example", + "scope_b": "example" }, { "skill_a": "broken-yaml-skill", @@ -4496,7 +10292,10 @@ "path_a": "tests/fixtures/package_invalid_yaml", "path_b": "tests/fixtures/package_missing_interface_field", "score": 0.833, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "test-fixture", + "scope_b": "test-fixture" }, { "skill_a": "frontend-review", @@ -4504,9 +10303,13 @@ "path_a": "examples/evolution-frontend-review/final", "path_b": "examples/evolution-frontend-review/v0", "score": 1.0, - "status": "duplicate-name" + "status": "duplicate-name", + "actionable": false, + "scope_a": "example", + "scope_b": "example" } ], + "actionable_route_collisions": [], "dependency_graph": { "nodes": [ { @@ -4564,56 +10367,77 @@ { "name": "frontend-review", "path": "examples/evolution-frontend-review/final", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/evolution-frontend-review/v0", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/evolution-frontend-review/v1", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", "age_days": 74, - "allowed_days": 31 + "allowed_days": 31, + "actionable": false, + "scope": "example" }, { "name": "note-cleanup", "path": "examples/simple-note-cleanup/generated-skill", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/team-frontend-review/generated-skill", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "geo-ranking-article-generator", "path": "geo-ranking-article-generator", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "embedded-generated-skill" }, { "name": "invalid-governance-skill", "path": "tests/fixtures/governance_invalid_manifest", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-yaml-skill", "path": "tests/fixtures/package_invalid_yaml", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-skill", "path": "tests/fixtures/package_missing_interface_field", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" } ], + "actionable_stale_skills": [], "owner_review_gaps": [ { "name": "frontend-review", @@ -4622,7 +10446,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -4631,7 +10457,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -4640,7 +10468,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "note-cleanup", @@ -4649,7 +10479,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -4658,7 +10490,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "geo-ranking-article-generator", @@ -4667,7 +10501,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "embedded-generated-skill" }, { "name": "invalid-governance-skill", @@ -4675,7 +10511,9 @@ "missing": [ "owner", "review_cadence" - ] + ], + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-yaml-skill", @@ -4684,7 +10522,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-skill", @@ -4693,9 +10533,12 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "test-fixture" } ], + "actionable_owner_review_gaps": [], "no_route_opportunities": [], "artifacts": { "catalog": "skill_atlas/catalog.json", @@ -4708,6 +10551,968 @@ "report_html": "reports/skill_atlas.html" } }, + "adoption_drift": { + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13T09:57:51Z", + "skill_dir": ".", + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "summary": { + "event_count": 1, + "activation_count": 1, + "accepted_count": 1, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 100.0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "low", + "event_types": { + "skill_activation": 1 + }, + "failure_types": {} + }, + "adoption_by_skill": [ + { + "skill": "yao-meta-skill", + "events": 1, + "accepted": 1, + "edited": 0, + "rejected": 0, + "missed": 0, + "adoption_rate": 100.0 + } + ], + "next_iteration_candidates": [], + "recent_events": [ + { + "event": "skill_activation", + "skill": "yao-meta-skill", + "version": "1.1.0", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-13T10:00:00Z" + } + ], + "failures": [], + "artifacts": { + "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", + "json": "reports/adoption_drift_report.json", + "markdown": "reports/adoption_drift_report.md" + } + }, + "review_waivers": { + "schema_version": "1.0", + "ok": true, + "skill_dir": ".", + "generated_at": "2026-06-13", + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [] + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "waivers": [], + "failures": [], + "warnings": [], + "artifacts": { + "json": "reports/review_waivers.json", + "markdown": "reports/review_waivers.md" + } + }, + "review_annotations": { + "schema_version": "1.0", + "ok": true, + "skill_dir": ".", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [], + "template_written": false, + "artifacts": { + "json": "reports/review_annotations.json", + "markdown": "reports/review_annotations.md" + } + }, + "registry": { + "ok": true, + "schema_version": "2.0", + "skill_dir": ".", + "registry_dir": "registry", + "package": { + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe", + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "agent-skills": "pass", + "vscode": "pass", + "generic": "pass", + "agent-skills-compatible": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "package_verification": "reports/package_verification.json", + "install_simulated": true, + "install_simulation": "reports/install_simulation.json" + }, + "generated_at": "2026-06-13" + }, + "index": { + "schema_version": "2.0", + "generated_at": "2026-06-13", + "package_count": 1, + "packages": [ + { + "name": "yao-meta-skill", + "version": "1.1.0", + "maturity": "governed", + "owner": "Yao Team", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "package_metadata": "registry/packages/yao-meta-skill.json", + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" + } + ] + }, + "failures": [], + "warnings": [], + "artifacts": { + "index": "registry/index.json", + "package": "registry/packages/yao-meta-skill.json", + "json": "reports/registry_audit.json", + "markdown": "reports/registry_audit.md" + } + }, + "package_verification": { + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13", + "skill_dir": ".", + "package_dir": "dist", + "summary": { + "target_count": 3, + "adapter_count": 3, + "archive_present": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "archive_entry_count": 483, + "failure_count": 0, + "warning_count": 0 + }, + "checks": [ + { + "id": "package-manifest", + "status": "pass", + "detail": "Package manifest exists: dist/manifest.json" + }, + { + "id": "openai-adapter", + "status": "pass", + "detail": "Adapter exists for target: openai" + }, + { + "id": "openai-field-name", + "status": "pass", + "detail": "openai adapter includes field: name" + }, + { + "id": "openai-field-description", + "status": "pass", + "detail": "openai adapter includes field: description" + }, + { + "id": "openai-field-version", + "status": "pass", + "detail": "openai adapter includes field: version" + }, + { + "id": "openai-field-display_name", + "status": "pass", + "detail": "openai adapter includes field: display_name" + }, + { + "id": "openai-field-short_description", + "status": "pass", + "detail": "openai adapter includes field: short_description" + }, + { + "id": "openai-field-default_prompt", + "status": "pass", + "detail": "openai adapter includes field: default_prompt" + }, + { + "id": "openai-field-job_to_be_done", + "status": "pass", + "detail": "openai adapter includes field: job_to_be_done" + }, + { + "id": "openai-field-ir_source", + "status": "pass", + "detail": "openai adapter includes field: ir_source" + }, + { + "id": "openai-field-ir_schema_version", + "status": "pass", + "detail": "openai adapter includes field: ir_schema_version" + }, + { + "id": "openai-field-semantic_contract", + "status": "pass", + "detail": "openai adapter includes field: semantic_contract" + }, + { + "id": "openai-field-semantic_parity", + "status": "pass", + "detail": "openai adapter includes field: semantic_parity" + }, + { + "id": "openai-field-compiler", + "status": "pass", + "detail": "openai adapter includes field: compiler" + }, + { + "id": "openai-field-compiled_contract", + "status": "pass", + "detail": "openai adapter includes field: compiled_contract" + }, + { + "id": "openai-field-permission_contract", + "status": "pass", + "detail": "openai adapter includes field: permission_contract" + }, + { + "id": "openai-field-target_permission_contract", + "status": "pass", + "detail": "openai adapter includes field: target_permission_contract" + }, + { + "id": "openai-field-target_native_contract", + "status": "pass", + "detail": "openai adapter includes field: target_native_contract" + }, + { + "id": "openai-field-target_transform", + "status": "pass", + "detail": "openai adapter includes field: target_transform" + }, + { + "id": "openai-field-canonical_metadata", + "status": "pass", + "detail": "openai adapter includes field: canonical_metadata" + }, + { + "id": "openai-field-canonical_format", + "status": "pass", + "detail": "openai adapter includes field: canonical_format" + }, + { + "id": "openai-field-activation_mode", + "status": "pass", + "detail": "openai adapter includes field: activation_mode" + }, + { + "id": "openai-field-execution_context", + "status": "pass", + "detail": "openai adapter includes field: execution_context" + }, + { + "id": "openai-field-shell", + "status": "pass", + "detail": "openai adapter includes field: shell" + }, + { + "id": "openai-field-trust_level", + "status": "pass", + "detail": "openai adapter includes field: trust_level" + }, + { + "id": "openai-field-remote_inline_execution", + "status": "pass", + "detail": "openai adapter includes field: remote_inline_execution" + }, + { + "id": "openai-field-degradation_strategy", + "status": "pass", + "detail": "openai adapter includes field: degradation_strategy" + }, + { + "id": "openai-field-portability_profile", + "status": "pass", + "detail": "openai adapter includes field: portability_profile" + }, + { + "id": "claude-adapter", + "status": "pass", + "detail": "Adapter exists for target: claude" + }, + { + "id": "claude-field-name", + "status": "pass", + "detail": "claude adapter includes field: name" + }, + { + "id": "claude-field-description", + "status": "pass", + "detail": "claude adapter includes field: description" + }, + { + "id": "claude-field-version", + "status": "pass", + "detail": "claude adapter includes field: version" + }, + { + "id": "claude-field-display_name", + "status": "pass", + "detail": "claude adapter includes field: display_name" + }, + { + "id": "claude-field-short_description", + "status": "pass", + "detail": "claude adapter includes field: short_description" + }, + { + "id": "claude-field-default_prompt", + "status": "pass", + "detail": "claude adapter includes field: default_prompt" + }, + { + "id": "claude-field-job_to_be_done", + "status": "pass", + "detail": "claude adapter includes field: job_to_be_done" + }, + { + "id": "claude-field-ir_source", + "status": "pass", + "detail": "claude adapter includes field: ir_source" + }, + { + "id": "claude-field-ir_schema_version", + "status": "pass", + "detail": "claude adapter includes field: ir_schema_version" + }, + { + "id": "claude-field-semantic_contract", + "status": "pass", + "detail": "claude adapter includes field: semantic_contract" + }, + { + "id": "claude-field-semantic_parity", + "status": "pass", + "detail": "claude adapter includes field: semantic_parity" + }, + { + "id": "claude-field-compiler", + "status": "pass", + "detail": "claude adapter includes field: compiler" + }, + { + "id": "claude-field-compiled_contract", + "status": "pass", + "detail": "claude adapter includes field: compiled_contract" + }, + { + "id": "claude-field-permission_contract", + "status": "pass", + "detail": "claude adapter includes field: permission_contract" + }, + { + "id": "claude-field-target_permission_contract", + "status": "pass", + "detail": "claude adapter includes field: target_permission_contract" + }, + { + "id": "claude-field-target_native_contract", + "status": "pass", + "detail": "claude adapter includes field: target_native_contract" + }, + { + "id": "claude-field-target_transform", + "status": "pass", + "detail": "claude adapter includes field: target_transform" + }, + { + "id": "claude-field-canonical_metadata", + "status": "pass", + "detail": "claude adapter includes field: canonical_metadata" + }, + { + "id": "claude-field-canonical_format", + "status": "pass", + "detail": "claude adapter includes field: canonical_format" + }, + { + "id": "claude-field-activation_mode", + "status": "pass", + "detail": "claude adapter includes field: activation_mode" + }, + { + "id": "claude-field-execution_context", + "status": "pass", + "detail": "claude adapter includes field: execution_context" + }, + { + "id": "claude-field-shell", + "status": "pass", + "detail": "claude adapter includes field: shell" + }, + { + "id": "claude-field-trust_level", + "status": "pass", + "detail": "claude adapter includes field: trust_level" + }, + { + "id": "claude-field-remote_inline_execution", + "status": "pass", + "detail": "claude adapter includes field: remote_inline_execution" + }, + { + "id": "claude-field-degradation_strategy", + "status": "pass", + "detail": "claude adapter includes field: degradation_strategy" + }, + { + "id": "claude-field-portability_profile", + "status": "pass", + "detail": "claude adapter includes field: portability_profile" + }, + { + "id": "generic-adapter", + "status": "pass", + "detail": "Adapter exists for target: generic" + }, + { + "id": "generic-field-name", + "status": "pass", + "detail": "generic adapter includes field: name" + }, + { + "id": "generic-field-description", + "status": "pass", + "detail": "generic adapter includes field: description" + }, + { + "id": "generic-field-version", + "status": "pass", + "detail": "generic adapter includes field: version" + }, + { + "id": "generic-field-display_name", + "status": "pass", + "detail": "generic adapter includes field: display_name" + }, + { + "id": "generic-field-short_description", + "status": "pass", + "detail": "generic adapter includes field: short_description" + }, + { + "id": "generic-field-default_prompt", + "status": "pass", + "detail": "generic adapter includes field: default_prompt" + }, + { + "id": "generic-field-job_to_be_done", + "status": "pass", + "detail": "generic adapter includes field: job_to_be_done" + }, + { + "id": "generic-field-ir_source", + "status": "pass", + "detail": "generic adapter includes field: ir_source" + }, + { + "id": "generic-field-ir_schema_version", + "status": "pass", + "detail": "generic adapter includes field: ir_schema_version" + }, + { + "id": "generic-field-semantic_contract", + "status": "pass", + "detail": "generic adapter includes field: semantic_contract" + }, + { + "id": "generic-field-semantic_parity", + "status": "pass", + "detail": "generic adapter includes field: semantic_parity" + }, + { + "id": "generic-field-compiler", + "status": "pass", + "detail": "generic adapter includes field: compiler" + }, + { + "id": "generic-field-compiled_contract", + "status": "pass", + "detail": "generic adapter includes field: compiled_contract" + }, + { + "id": "generic-field-permission_contract", + "status": "pass", + "detail": "generic adapter includes field: permission_contract" + }, + { + "id": "generic-field-target_permission_contract", + "status": "pass", + "detail": "generic adapter includes field: target_permission_contract" + }, + { + "id": "generic-field-target_native_contract", + "status": "pass", + "detail": "generic adapter includes field: target_native_contract" + }, + { + "id": "generic-field-target_transform", + "status": "pass", + "detail": "generic adapter includes field: target_transform" + }, + { + "id": "generic-field-canonical_metadata", + "status": "pass", + "detail": "generic adapter includes field: canonical_metadata" + }, + { + "id": "generic-field-canonical_format", + "status": "pass", + "detail": "generic adapter includes field: canonical_format" + }, + { + "id": "generic-field-activation_mode", + "status": "pass", + "detail": "generic adapter includes field: activation_mode" + }, + { + "id": "generic-field-execution_context", + "status": "pass", + "detail": "generic adapter includes field: execution_context" + }, + { + "id": "generic-field-shell", + "status": "pass", + "detail": "generic adapter includes field: shell" + }, + { + "id": "generic-field-trust_level", + "status": "pass", + "detail": "generic adapter includes field: trust_level" + }, + { + "id": "generic-field-remote_inline_execution", + "status": "pass", + "detail": "generic adapter includes field: remote_inline_execution" + }, + { + "id": "generic-field-degradation_strategy", + "status": "pass", + "detail": "generic adapter includes field: degradation_strategy" + }, + { + "id": "generic-field-portability_profile", + "status": "pass", + "detail": "generic adapter includes field: portability_profile" + }, + { + "id": "openai-file-targets/openai/adapter.json", + "status": "pass", + "detail": "Package contains targets/openai/adapter.json" + }, + { + "id": "openai-file-targets/openai/agents/openai.yaml", + "status": "pass", + "detail": "Package contains targets/openai/agents/openai.yaml" + }, + { + "id": "claude-file-targets/claude/adapter.json", + "status": "pass", + "detail": "Package contains targets/claude/adapter.json" + }, + { + "id": "claude-file-targets/claude/README.md", + "status": "pass", + "detail": "Package contains targets/claude/README.md" + }, + { + "id": "generic-file-targets/generic/adapter.json", + "status": "pass", + "detail": "Package contains targets/generic/adapter.json" + }, + { + "id": "archive-safe-paths", + "status": "pass", + "detail": "Archive has no absolute or parent-traversal entries" + }, + { + "id": "archive-entry-yao-meta-skill/SKILL.md", + "status": "pass", + "detail": "Archive contains yao-meta-skill/SKILL.md" + }, + { + "id": "archive-entry-yao-meta-skill/manifest.json", + "status": "pass", + "detail": "Archive contains yao-meta-skill/manifest.json" + }, + { + "id": "archive-entry-yao-meta-skill/agents/interface.yaml", + "status": "pass", + "detail": "Archive contains yao-meta-skill/agents/interface.yaml" + }, + { + "id": "archive-excludes-generated", + "status": "pass", + "detail": "Archive excludes generated dist/ and tests/tmp* contents" + }, + { + "id": "registry-ok", + "status": "pass", + "detail": "Registry audit is OK" + }, + { + "id": "registry-name-match", + "status": "pass", + "detail": "Registry package name matches package manifest" + }, + { + "id": "registry-version-match", + "status": "pass", + "detail": "Registry package version matches package manifest" + }, + { + "id": "registry-compat-openai", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: openai" + }, + { + "id": "registry-compat-claude", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: claude" + }, + { + "id": "registry-compat-generic", + "status": "pass", + "detail": "Registry compatibility is reviewable for target: generic" + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "manifest": "dist/manifest.json", + "archive": "dist/yao-meta-skill.zip", + "json": "reports/package_verification.json", + "markdown": "reports/package_verification.md" + } + }, + "install_simulation": { + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13", + "skill_dir": ".", + "package_dir": "dist", + "install_root": "dist/install-simulation/simulate-yao-meta-skill", + "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "summary": { + "archive_present": true, + "archive_entry_count": 483, + "archive_extracted": true, + "entrypoint_loaded": true, + "manifest_loaded": true, + "interface_loaded": true, + "adapter_count": 3, + "install_root_is_temp": false, + "failure_count": 0, + "warning_count": 0 + }, + "checks": [ + { + "id": "archive-present", + "status": "pass", + "detail": "Package archive exists: dist/yao-meta-skill.zip" + }, + { + "id": "archive-safe-paths", + "status": "pass", + "detail": "Archive has no absolute or parent-traversal entries" + }, + { + "id": "single-top-level", + "status": "pass", + "detail": "Archive top-level directory is yao-meta-skill" + }, + { + "id": "entrypoint-load", + "status": "pass", + "detail": "Installed SKILL.md frontmatter is readable" + }, + { + "id": "entrypoint-name", + "status": "pass", + "detail": "Installed SKILL.md name matches package directory" + }, + { + "id": "entrypoint-description", + "status": "pass", + "detail": "Installed SKILL.md description is present" + }, + { + "id": "manifest-load", + "status": "pass", + "detail": "Installed manifest.json is readable" + }, + { + "id": "manifest-name", + "status": "pass", + "detail": "Installed manifest name matches package manifest" + }, + { + "id": "manifest-version", + "status": "pass", + "detail": "Installed manifest version matches package manifest" + }, + { + "id": "interface-load", + "status": "pass", + "detail": "Installed agents/interface.yaml is readable" + }, + { + "id": "overview-report", + "status": "pass", + "detail": "Installed overview report is present" + }, + { + "id": "review-studio-report", + "status": "pass", + "detail": "Installed Review Studio report is present" + }, + { + "id": "adapter-claude", + "status": "pass", + "detail": "claude adapter is readable after package install simulation" + }, + { + "id": "adapter-claude-name", + "status": "pass", + "detail": "claude adapter name matches package manifest" + }, + { + "id": "adapter-generic", + "status": "pass", + "detail": "generic adapter is readable after package install simulation" + }, + { + "id": "adapter-generic-name", + "status": "pass", + "detail": "generic adapter name matches package manifest" + }, + { + "id": "adapter-openai", + "status": "pass", + "detail": "openai adapter is readable after package install simulation" + }, + { + "id": "adapter-openai-name", + "status": "pass", + "detail": "openai adapter name matches package manifest" + } + ], + "failures": [], + "warnings": [], + "artifacts": { + "archive": "dist/yao-meta-skill.zip", + "package_manifest": "dist/manifest.json", + "json": "reports/install_simulation.json", + "markdown": "reports/install_simulation.md" + } + }, + "upgrade_check": { + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13", + "previous": { + "name": "yao-meta-skill", + "version": "1.0.0", + "targets": [ + "openai", + "claude", + "generic" + ] + }, + "current": { + "name": "yao-meta-skill", + "version": "1.1.0", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "summary": { + "declared_bump": "minor", + "recommended_bump": "minor", + "breaking_change_count": 0, + "added_target_count": 1, + "metadata_change_count": 1, + "checksum_change_count": 2, + "failure_count": 0, + "warning_count": 0 + }, + "upgrade_diff": { + "name_changed": false, + "added_targets": [ + "agent-skills-compatible" + ], + "removed_targets": [], + "compatibility_changes": [ + { + "target": "agent-skills", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "agent-skills-compatible", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "vscode", + "from": "missing", + "to": "pass", + "direction": "improved" + } + ], + "metadata_changes": [ + { + "field": "description", + "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + } + ], + "checksum_changes": [ + { + "field": "archive_sha256", + "from": "", + "to": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + { + "field": "package_sha256", + "from": "0000000000000000000000000000000000000000000000000000000000000000", + "to": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" + } + ] + }, + "release_notes": [ + "Recommended version bump: minor.", + "Added targets: agent-skills-compatible.", + "Package or archive checksum changed; reviewers should verify package artifacts before release.", + "Package verification evidence: reports/package_verification.md." + ], + "failures": [], + "warnings": [], + "artifacts": { + "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", + "current_package": "reports/registry_audit.json", + "json": "reports/upgrade_check.json", + "markdown": "reports/upgrade_check.md" + } + }, "manifest": { "name": "yao-meta-skill", "version": "1.1.0", diff --git a/reports/review_annotations.json b/reports/review_annotations.json new file mode 100644 index 0000000..66f2b1d --- /dev/null +++ b/reports/review_annotations.json @@ -0,0 +1,24 @@ +{ + "schema_version": "1.0", + "ok": true, + "skill_dir": ".", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [], + "template_written": false, + "artifacts": { + "json": "reports/review_annotations.json", + "markdown": "reports/review_annotations.md" + } +} diff --git a/reports/review_annotations.md b/reports/review_annotations.md new file mode 100644 index 0000000..68230eb --- /dev/null +++ b/reports/review_annotations.md @@ -0,0 +1,19 @@ +# Review Annotations + +This report renders reviewer annotations attached to Review Studio gates and source/report paths. + +- Annotations: `0` +- Open: `0` +- Resolved: `0` +- Deferred: `0` +- Open blockers: `0` +- Open warnings: `0` + +No reviewer annotations recorded yet. + + +## Review Rule + +- Use annotations for reviewer comments tied to a gate or source line. +- Use waivers only for explicit acceptance of warning-level release risk. +- Open blocker annotations should block a release decision until resolved or deferred with rationale. diff --git a/reports/review_waivers.json b/reports/review_waivers.json new file mode 100644 index 0000000..c3a9d45 --- /dev/null +++ b/reports/review_waivers.json @@ -0,0 +1,40 @@ +{ + "schema_version": "1.0", + "ok": true, + "skill_dir": ".", + "generated_at": "2026-06-13", + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [] + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "waivers": [], + "failures": [], + "warnings": [], + "artifacts": { + "json": "reports/review_waivers.json", + "markdown": "reports/review_waivers.md" + } +} diff --git a/reports/review_waivers.md b/reports/review_waivers.md new file mode 100644 index 0000000..dc684bb --- /dev/null +++ b/reports/review_waivers.md @@ -0,0 +1,26 @@ +# Review Waivers + +- OK: `True` +- Waivers: `0` +- Active: `0` +- Expired: `0` +- Invalid: `0` +- Covered gates: `none` + +## Policy + +- Blocker waivers allowed: `False` +- Minimum reason chars: `20` +- Expiry is required for every waiver. + +## Waivers + +- None + +## Failures + +- None + +## Warnings + +- None diff --git a/reports/runtime_permission_probes.json b/reports/runtime_permission_probes.json new file mode 100644 index 0000000..e68842e --- /dev/null +++ b/reports/runtime_permission_probes.json @@ -0,0 +1,253 @@ +{ + "schema_version": "1.0", + "ok": true, + "skill_dir": ".", + "package_dir": "tests/tmp_review_studio/dist", + "expected_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "summary": { + "target_count": 3, + "pass_count": 3, + "fail_count": 0, + "native_enforcement_count": 0, + "metadata_fallback_count": 3, + "residual_risk_count": 3, + "required_capability_count": 3, + "failure_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", + "permission_model": "metadata-only", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "openai adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "openai adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "openai adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "openai permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "openai target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "openai target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "openai target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "openai target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "openai target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "openai review_required matches whether capabilities are required" + }, + { + "key": "openai-yaml-present", + "passed": true, + "detail": "OpenAI permission metadata YAML is readable" + }, + { + "key": "openai-yaml-permissions", + "passed": true, + "detail": "OpenAI YAML permission contract mirrors expected capabilities" + }, + { + "key": "openai-yaml-native-flag", + "passed": true, + "detail": "OpenAI YAML declares native_enforcement as a boolean" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "claude", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", + "permission_model": "neutral-source-plus-adapter", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "claude adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "claude adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "claude adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "claude permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "claude target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "claude target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "claude target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "claude target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "claude target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "claude review_required matches whether capabilities are required" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "generic", + "status": "pass", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", + "permission_model": "agent-skills-compatible-metadata", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "assurance": "metadata-fallback-explicit", + "declared_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "checks": [ + { + "key": "adapter-present", + "passed": true, + "detail": "generic adapter.json is readable" + }, + { + "key": "permission-contract-present", + "passed": true, + "detail": "generic adapter includes permission_contract" + }, + { + "key": "target-contract-present", + "passed": true, + "detail": "generic adapter includes target_permission_contract" + }, + { + "key": "source-available", + "passed": true, + "detail": "generic permission_contract links to an available trust report" + }, + { + "key": "declared-capabilities-match", + "passed": true, + "detail": "generic target_permission_contract mirrors expected capabilities" + }, + { + "key": "capability-counts-present", + "passed": true, + "detail": "generic target_permission_contract includes capability_counts" + }, + { + "key": "native-enforcement-boolean", + "passed": true, + "detail": "generic target_permission_contract declares native_enforcement as a boolean" + }, + { + "key": "representation-present", + "passed": true, + "detail": "generic target_permission_contract declares where permission metadata is represented" + }, + { + "key": "operator-note-present", + "passed": true, + "detail": "generic target_permission_contract includes an operator_note" + }, + { + "key": "review-required-matches", + "passed": true, + "detail": "generic review_required matches whether capabilities are required" + } + ], + "failures": [], + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + } + ], + "failures": [], + "artifacts": { + "json": "reports/runtime_permission_probes.json", + "markdown": "reports/runtime_permission_probes.md" + } +} diff --git a/reports/runtime_permission_probes.md b/reports/runtime_permission_probes.md new file mode 100644 index 0000000..1e68c33 --- /dev/null +++ b/reports/runtime_permission_probes.md @@ -0,0 +1,27 @@ +# Runtime Permission Probes + +Runtime permission probes verify that generated target adapters expose high-permission capabilities and make native-enforcement limits explicit. + +## Summary + +- OK: `True` +- Targets probed: `3` +- Passed: `3` +- Failed: `0` +- Native enforcement targets: `0` +- Explicit metadata fallbacks: `3` +- Required capabilities: `file_write, network, subprocess` + +| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Residual Risk | +| --- | --- | --- | --- | --- | --- | +| `openai` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | +| `claude` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | +| `generic` | `pass` | `metadata-fallback-explicit` | `False` | `True` | Client-native permission enforcement is not provided by this target; installer or operator must honor metadata. | + +## Failures + +- None + +## Reviewer Note + +A passing probe means the target contract is explicit and auditable. It does not claim that a host client enforces permissions natively. diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index aca4c7e..1317dd6 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,422 +2,913 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 127, - "script_count": 50, + "scanned_files": 146, + "script_count": 62, + "internal_module_count": 3, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], "network_script_count": 2, + "network_policy_covered_count": 2, + "network_policy_missing_count": 0, + "file_write_script_count": 47, + "permission_required_count": 3, + "permission_approved_count": 3, + "permission_missing_count": 0, + "permission_invalid_count": 0, + "permission_expired_count": 0, + "help_smoke_checked_count": 59, + "help_smoke_failed_count": 0, "interactive_script_count": 0, - "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" + "package_hash_scope": "source-contract-without-generated-reports", + "package_hash_file_count": 146, + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" }, "failures": [], - "warnings": [ - "Scripts without argparse/help surface: scripts/render_context_reports.py, scripts/render_social_preview.py, scripts/skill_report_charts.py, scripts/skill_report_metrics.py, scripts/skill_report_model.py", - "Network-capable scripts require bounded host policy: scripts/check_update.py, scripts/github_benchmark_scan.py" - ], + "warnings": [], "secrets": [], "scripts": [ { - "path": "scripts/build_confusion_matrix.py", + "path": "scripts/adjudicate_output_review.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/build_confusion_matrix.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/build_skill_atlas.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/check_update.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": true, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [ + "https://raw.githubusercontent.com/yaojingang/yao-meta-skill/main/VERSION", + "https://raw.githubusercontent.com/yaojingang/yao-meta-skill/main/manifest.json" + ], + "network_hosts": [ + "raw.githubusercontent.com" + ] }, { "path": "scripts/ci_test.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/collect_feedback.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/compile_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/context_sizer.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/create_iteration_snapshot.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/cross_packager.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/diff_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/export_skill_ir.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/github_benchmark_scan.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": true, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [ + "https://api.github.com" + ], + "network_hosts": [ + "api.github.com" + ] }, { "path": "scripts/governance_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/init_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/judge_blind_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/lint_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/local_output_eval_runner.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/optimize_description.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/probe_runtime_permissions.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/promotion_checker.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/registry_audit.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_adoption_drift_report.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_artifact_design_profile.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_baseline_compare.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_context_reports.py", - "has_argparse": false, + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_description_drift_history.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_eval_dashboard.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_intent_confidence.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_intent_dialogue.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_iteration_directions.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_iteration_ledger.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_output_risk_profile.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_portability_report.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_prompt_quality_profile.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_reference_scan.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_reference_synthesis.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_regression_history.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_review_annotations.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_review_studio.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_review_viewer.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/render_review_waivers.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_skill_overview.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/render_social_preview.py", - "has_argparse": false, + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [ + "http://www.w3.org/2000/svg" + ], + "network_hosts": [ + "www.w3.org" + ] }, { "path": "scripts/render_system_model.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/resource_boundary_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_conformance_suite.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_description_optimization_suite.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_eval_suite.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/run_output_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/run_output_execution.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/simulate_install.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/skill_report_charts.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_overview.py to render inline SVG report charts.", "has_argparse": false, "has_main_guard": false, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/skill_report_metrics.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by skill_report_model.py to calculate overview report metrics.", "has_argparse": false, "has_main_guard": false, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/skill_report_model.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by render_skill_overview.py to build the v2 report data model.", "has_argparse": false, "has_main_guard": false, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/sync_local_install.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/trigger_eval.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/trust_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/upgrade_check.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/validate_skill.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": false + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, + { + "path": "scripts/verify_package.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] }, { "path": "scripts/yao.py", + "interface": "cli", + "interface_declared": false, + "interface_reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules.", "has_argparse": true, "has_main_guard": true, "uses_input": false, "uses_network": false, - "uses_subprocess": true + "uses_file_write": false, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] } ], "dependencies": { @@ -429,6 +920,782 @@ ], "unpinned": [] }, + "network_policy": { + "present": true, + "path": "security/network_policy.json", + "schema_version": "1.0", + "network_script_count": 2, + "covered_scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "missing_scripts": [], + "mismatches": [], + "default_policy": { + "require_https": true, + "custom_hosts": "deny-by-default", + "review_rule": "Network-capable scripts must declare allowed_hosts and justify any explicit custom-host override." + } + }, + "help_smoke": { + "enabled": true, + "timeout_seconds": 5.0, + "candidate_count": 59, + "checked_count": 59, + "passed_count": 59, + "failed_count": 0, + "skipped_count": 3, + "failed_scripts": [], + "results": [ + { + "path": "scripts/adjudicate_output_review.py", + "command": "python3 scripts/adjudicate_output_review.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: adjudicate_output_review.py [-h] [--blind-pack BLIND_PACK]\n [--answer-key ANSWER_KEY]\n [--decisions DECISIONS]\n [--output-json OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/build_confusion_matrix.py", + "command": "python3 scripts/build_confusion_matrix.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: build_confusion_matrix.py [-h] [--cases CASES]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--history-snapshot-output HISTORY_", + "stderr_excerpt": "" + }, + { + "path": "scripts/build_skill_atlas.py", + "command": "python3 scripts/build_skill_atlas.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: build_skill_atlas.py [-h] [--workspace-root WORKSPACE_ROOT]\n [--output-dir OUTPUT_DIR]\n [--report-html REPORT_HTML]\n [--report-json REPORT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/check_update.py", + "command": "python3 scripts/check_update.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: check_update.py [-h] [--root ROOT] [--cache-path CACHE_PATH]\n [--version-url VERSION_URL]\n [--manifest-url MANIFEST_URL] [--timeout TIMEOUT]\n [--max-age-days MAX_AGE_D", + "stderr_excerpt": "" + }, + { + "path": "scripts/ci_test.py", + "command": "python3 scripts/ci_test.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: ci_test.py [-h] [--tail-lines TAIL_LINES] [targets ...]\n\nRun CI test targets with compact logging.\n\npositional arguments:\n targets Optional target override.\n\noptions:\n -h, --help show this help message and ", + "stderr_excerpt": "" + }, + { + "path": "scripts/collect_feedback.py", + "command": "python3 scripts/collect_feedback.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: collect_feedback.py [-h] [--note NOTE] [--rating RATING]\n [--category CATEGORY]\n [--recommended-action RECOMMENDED_ACTION]\n [--output-json OUTPUT_JSON] [--", + "stderr_excerpt": "" + }, + { + "path": "scripts/compile_skill.py", + "command": "python3 scripts/compile_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: compile_skill.py [-h] [--target TARGET] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD] [--generated-at GENERATED_AT]\n [skill_dir]\n\nCompile Skill IR into target-specific semantic con", + "stderr_excerpt": "" + }, + { + "path": "scripts/context_sizer.py", + "command": "python3 scripts/context_sizer.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: context_sizer.py [-h] [--json] skill_dir\n\nEstimate context size for a skill package.\n\npositional arguments:\n skill_dir Path to the skill directory\n\noptions:\n -h, --help show this help message and exit\n --json Emit machine-", + "stderr_excerpt": "" + }, + { + "path": "scripts/create_iteration_snapshot.py", + "command": "python3 scripts/create_iteration_snapshot.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: create_iteration_snapshot.py [-h] --target TARGET [--label LABEL]\n [--promotion-decisions PROMOTION_DECISIONS]\n [--route-scorecard ROUTE_SCORECARD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/cross_packager.py", + "command": "python3 scripts/cross_packager.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: cross_packager.py [-h] [--platform PLATFORM] [--output-dir OUTPUT_DIR]\n [--expectations EXPECTATIONS] [--zip]\n skill_dir\n\nGenerate lightweight cross-platform packaging artifacts.\n\nposit", + "stderr_excerpt": "" + }, + { + "path": "scripts/diff_eval.py", + "command": "python3 scripts/diff_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: diff_eval.py [-h] before after\n\nDiff two eval reports.\n\npositional arguments:\n before\n after\n\noptions:\n -h, --help show this help message and exit", + "stderr_excerpt": "" + }, + { + "path": "scripts/export_skill_ir.py", + "command": "python3 scripts/export_skill_ir.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: export_skill_ir.py [-h] [--output-json OUTPUT_JSON] [--validate-only]\n [skill_dir]\n\nExport a platform-neutral Skill IR document from a skill package.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help ", + "stderr_excerpt": "" + }, + { + "path": "scripts/github_benchmark_scan.py", + "command": "python3 scripts/github_benchmark_scan.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: github_benchmark_scan.py [-h] --query QUERY [--top-n TOP_N]\n [--fixture-dir FIXTURE_DIR]\n [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSO", + "stderr_excerpt": "" + }, + { + "path": "scripts/governance_check.py", + "command": "python3 scripts/governance_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: governance_check.py [-h] [--require-manifest] skill_dir\n\nCheck skill governance metadata and lifecycle readiness.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit\n --require-manifest", + "stderr_excerpt": "" + }, + { + "path": "scripts/init_skill.py", + "command": "python3 scripts/init_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: init_skill.py [-h] [--description DESCRIPTION] [--title TITLE]\n [--output-dir OUTPUT_DIR]\n [--mode {governed,library,production,scaffold}]\n [--archetype {governed,library,pr", + "stderr_excerpt": "" + }, + { + "path": "scripts/judge_blind_eval.py", + "command": "python3 scripts/judge_blind_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: judge_blind_eval.py [-h] [--description DESCRIPTION]\n [--description-file DESCRIPTION_FILE] --cases CASES\n [--semantic-config SEMANTIC_CONFIG]\n\nRun a rubric-based blind trigger judg", + "stderr_excerpt": "" + }, + { + "path": "scripts/lint_skill.py", + "command": "python3 scripts/lint_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: lint_skill.py [-h] skill_dir\n\nLint a skill package for basic structure issues.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit", + "stderr_excerpt": "" + }, + { + "path": "scripts/local_output_eval_runner.py", + "command": "python3 scripts/local_output_eval_runner.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: local_output_eval_runner.py [-h] [--execution-kind {command,model}]\n [--provider PROVIDER] [--model MODEL]\n\nLocal deterministic output-eval runner. It executes the runner contract\nwithout claiming pr", + "stderr_excerpt": "" + }, + { + "path": "scripts/optimize_description.py", + "command": "python3 scripts/optimize_description.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: optimize_description.py [-h] --description-file DESCRIPTION_FILE\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n --dev-cases DEV_CASES\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/probe_runtime_permissions.py", + "command": "python3 scripts/probe_runtime_permissions.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: probe_runtime_permissions.py [-h] [--package-dir PACKAGE_DIR]\n [--target {openai,claude,generic}]\n [--output-json OUTPUT_JSON]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/promotion_checker.py", + "command": "python3 scripts/promotion_checker.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: promotion_checker.py [-h] [--optimization-suite OPTIMIZATION_SUITE]\n [--route-scorecard ROUTE_SCORECARD]\n [--output-json OUTPUT_JSON]\n [--output-md OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/registry_audit.py", + "command": "python3 scripts/registry_audit.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: registry_audit.py [-h] [--registry-dir REGISTRY_DIR]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n [skill_dir]\n\nBuild and ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_adoption_drift_report.py", + "command": "python3 scripts/render_adoption_drift_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_adoption_drift_report.py [-h] [--events-jsonl EVENTS_JSONL]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_artifact_design_profile.py", + "command": "python3 scripts/render_artifact_design_profile.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_artifact_design_profile.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender artifact design direction and visual qua", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_baseline_compare.py", + "command": "python3 scripts/render_baseline_compare.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_baseline_compare.py [-h] --entry ENTRY\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a baseline comparison report from description optimization ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_context_reports.py", + "command": "python3 scripts/render_context_reports.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_context_reports.py [-h] [--generated-at GENERATED_AT]\n\nRender context budget reports for root and example skills.\n\noptions:\n -h, --help show this help message and exit\n --generated-at GENERATED_AT\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_description_drift_history.py", + "command": "python3 scripts/render_description_drift_history.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_description_drift_history.py [-h] [--history-dir HISTORY_DIR]\n [--output OUTPUT]\n\nRender description optimization drift history.\n\noptions:\n -h, --help show this help messag", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_eval_dashboard.py", + "command": "python3 scripts/render_eval_dashboard.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_eval_dashboard.py [-h] [--eval-dir EVAL_DIR]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_intent_confidence.py", + "command": "python3 scripts/render_intent_confidence.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_intent_confidence.py [-h] [--context-json CONTEXT_JSON]\n [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\n", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_intent_dialogue.py", + "command": "python3 scripts/render_intent_dialogue.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_intent_dialogue.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender an intent dialogue guide for a skill package.\n\npositional argume", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_iteration_directions.py", + "command": "python3 scripts/render_iteration_directions.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_iteration_directions.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender the top three next iteration directions for a ski", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_iteration_ledger.py", + "command": "python3 scripts/render_iteration_ledger.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_iteration_ledger.py [-h] [--history-dir HISTORY_DIR]\n [--description-history-dir DESCRIPTION_HISTORY_DIR]\n [--route-scorecard ROUTE_SCORECARD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_output_risk_profile.py", + "command": "python3 scripts/render_output_risk_profile.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_output_risk_profile.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender predicted output failure modes and quality constrain", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_portability_report.py", + "command": "python3 scripts/render_portability_report.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_portability_report.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a portability score from neutral metadata, contracts, and snapshots.\n\noptions:\n -h, --help ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_prompt_quality_profile.py", + "command": "python3 scripts/render_prompt_quality_profile.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_prompt_quality_profile.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender prompt behavior quality, RTF mapping, and n", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_reference_scan.py", + "command": "python3 scripts/render_reference_scan.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_reference_scan.py [-h] [--external-reference EXTERNAL_REFERENCE]\n [--user-reference USER_REFERENCE]\n [--local-constraint LOCAL_CONSTRAINT]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_reference_synthesis.py", + "command": "python3 scripts/render_reference_synthesis.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_reference_synthesis.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a multi-source reference synthesis report for a skil", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_regression_history.py", + "command": "python3 scripts/render_regression_history.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_regression_history.py [-h] [--history-dir HISTORY_DIR]\n [--output OUTPUT]\n\nRender regression history from milestone snapshots.\n\noptions:\n -h, --help show this help message and exi", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_annotations.py", + "command": "python3 scripts/render_review_annotations.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_annotations.py [-h] [--annotations-json ANNOTATIONS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD] [--write-template]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_studio.py", + "command": "python3 scripts/render_review_studio.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_studio.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender Review Studio 2.0 for a skill package.\n\npositional arguments:\n sk", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_viewer.py", + "command": "python3 scripts/render_review_viewer.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_viewer.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a compact HTML review viewer for a skill package.\n\npositional argu", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_review_waivers.py", + "command": "python3 scripts/render_review_waivers.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_review_waivers.py [-h] [--waivers-json WAIVERS_JSON]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_skill_overview.py", + "command": "python3 scripts/render_skill_overview.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_skill_overview.py [-h] [--output-html OUTPUT_HTML]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender the HTML skill report for a skill package.\n\npositional argument", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_social_preview.py", + "command": "python3 scripts/render_social_preview.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_social_preview.py [-h] [--format {all,png,svg}]\n\nRender the repository social preview image assets.\n\noptions:\n -h, --help show this help message and exit\n --format {all,png,svg}\n Asset forma", + "stderr_excerpt": "" + }, + { + "path": "scripts/render_system_model.py", + "command": "python3 scripts/render_system_model.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_system_model.py [-h] [--output-md OUTPUT_MD]\n [--output-json OUTPUT_JSON]\n [skill_dir]\n\nRender a systems-thinking model for a skill package.\n\npositional arguments:\n sk", + "stderr_excerpt": "" + }, + { + "path": "scripts/resource_boundary_check.py", + "command": "python3 scripts/resource_boundary_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: resource_boundary_check.py [-h]\n [--max-initial-tokens MAX_INITIAL_TOKENS]\n [--warn-skill-body-tokens WARN_SKILL_BODY_TOKENS]\n skill_d", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_conformance_suite.py", + "command": "python3 scripts/run_conformance_suite.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_conformance_suite.py [-h]\n [--target {openai,claude,agent-skills,vscode,generic}]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_description_optimization_suite.py", + "command": "python3 scripts/run_description_optimization_suite.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_description_optimization_suite.py [-h]\n [--history-snapshot-output HISTORY_SNAPSHOT_OUTPUT]\n [--snapshot-date SNAPSHOT_DATE]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_eval_suite.py", + "command": "python3 scripts/run_eval_suite.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_eval_suite.py [-h] [--eval-dir EVAL_DIR]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description-file BASELINE_DESCRIPTION_FILE]\n [--output-file OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_output_eval.py", + "command": "python3 scripts/run_output_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_output_eval.py [-h] [--cases CASES] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--blind-pack-json BLIND_PACK_JSON]\n [--blind-pack-md BLIND_PACK", + "stderr_excerpt": "" + }, + { + "path": "scripts/run_output_execution.py", + "command": "python3 scripts/run_output_execution.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: run_output_execution.py [-h] [--cases CASES]\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--runner-command RUNNER_COMMAND]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/simulate_install.py", + "command": "python3 scripts/simulate_install.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: simulate_install.py [-h] [--package-dir PACKAGE_DIR]\n [--install-root INSTALL_ROOT]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--generated-at G", + "stderr_excerpt": "" + }, + { + "path": "scripts/sync_local_install.py", + "command": "python3 scripts/sync_local_install.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: sync_local_install.py [-h] [--root ROOT] [--install-dir INSTALL_DIR]\n [--dry-run]\n\nSync the current yao-meta-skill source into a managed local skill mirror.\n\noptions:\n -h, --help show this help", + "stderr_excerpt": "" + }, + { + "path": "scripts/trigger_eval.py", + "command": "python3 scripts/trigger_eval.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: trigger_eval.py [-h] [--description DESCRIPTION]\n [--description-file DESCRIPTION_FILE]\n [--baseline-description BASELINE_DESCRIPTION]\n [--baseline-description-file BA", + "stderr_excerpt": "" + }, + { + "path": "scripts/trust_check.py", + "command": "python3 scripts/trust_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: trust_check.py [-h] [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n [--skip-help-smoke]\n [--help-smoke-timeout HELP_SMOKE_TIMEOUT]\n [skill_dir]\n\nRun Skill OS trust a", + "stderr_excerpt": "" + }, + { + "path": "scripts/upgrade_check.py", + "command": "python3 scripts/upgrade_check.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: upgrade_check.py [-h] --previous-package-json PREVIOUS_PACKAGE_JSON\n [--current-package-json CURRENT_PACKAGE_JSON]\n [--output-json OUTPUT_JSON] [--output-md OUTPUT_MD]\n ", + "stderr_excerpt": "" + }, + { + "path": "scripts/validate_skill.py", + "command": "python3 scripts/validate_skill.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: validate_skill.py [-h] skill_dir\n\nValidate a skill package.\n\npositional arguments:\n skill_dir\n\noptions:\n -h, --help show this help message and exit", + "stderr_excerpt": "" + }, + { + "path": "scripts/verify_package.py", + "command": "python3 scripts/verify_package.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: verify_package.py [-h] [--package-dir PACKAGE_DIR]\n [--expectations EXPECTATIONS]\n [--registry-json REGISTRY_JSON]\n [--output-json OUTPUT_JSON] [--output-md OUTP", + "stderr_excerpt": "" + }, + { + "path": "scripts/yao.py", + "command": "python3 scripts/yao.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: yao.py [-h]\n {init,quickstart,validate,optimize-description,promote-check,review,release-snapshot,workspace-flow,report,skill-report,review-viewer,review-studio,reference-scan,github-benchmark-scan,intent-confidence,inte", + "stderr_excerpt": "" + } + ], + "skipped": [ + { + "path": "scripts/skill_report_charts.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_metrics.py", + "reason": "internal module" + }, + { + "path": "scripts/skill_report_model.py", + "reason": "internal module" + } + ] + }, + "permission_governance": { + "present": true, + "path": "security/permission_policy.json", + "schema_version": "1.0", + "reviewed_at": "2026-06-13", + "required_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "approved_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "missing_capabilities": [], + "invalid_capabilities": [], + "expired_capabilities": [], + "approval_count": 3, + "required_count": 3, + "missing_count": 0, + "invalid_count": 0, + "expired_count": 0, + "approvals": [ + { + "capability": "network", + "status": "approved", + "scripts": [ + "scripts/check_update.py", + "scripts/github_benchmark_scan.py" + ], + "reviewer": "Yao Team", + "scope": "Only scripts declared in security/network_policy.json may perform outbound requests.", + "reason": "Network use is bounded to public GitHub metadata and benchmark discovery, with allowed hosts reviewed in security/network_policy.json.", + "expires_at": "2026-09-30", + "evidence": [ + "security/network_policy.json", + "reports/security_trust_report.md" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + }, + "validation": [] + }, + { + "capability": "file_write", + "status": "approved", + "scripts": [ + "scripts/adjudicate_output_review.py", + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/compile_skill.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/init_skill.py", + "scripts/probe_runtime_permissions.py", + "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/upgrade_check.py", + "scripts/verify_package.py" + ], + "reviewer": "Yao Team", + "scope": "Local generated reports, registry metadata, temporary test outputs, and package artifacts under the skill workspace.", + "reason": "File writes are required to create auditable reports, compiled target contracts, package artifacts, and local test fixtures; package safety checks exclude generated temp outputs from archives.", + "expires_at": "2026-09-30", + "evidence": [ + "reports/package_verification.md", + "reports/install_simulation.md", + "security/permission_policy.md" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + }, + "validation": [] + }, + { + "capability": "subprocess", + "status": "approved", + "scripts": [ + "scripts/ci_test.py", + "scripts/render_eval_dashboard.py", + "scripts/run_eval_suite.py", + "scripts/run_output_execution.py", + "scripts/sync_local_install.py", + "scripts/trust_check.py", + "scripts/yao.py" + ], + "reviewer": "Yao Team", + "scope": "Local deterministic validation, packaging, install simulation, and CI orchestration commands.", + "reason": "Subprocess use is limited to local verification and packaging flows that are surfaced through CLI help smoke tests and CI targets.", + "expires_at": "2026-09-30", + "evidence": [ + "reports/security_trust_report.md", + "reports/review-studio.html" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + }, + "validation": [] + } + ] + }, "trust_metadata": { "source_tier": "local", "remote_inline_execution": "forbid", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 00aa421..9d87392 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,12 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `127` -- Scripts: `50` +- Scanned files: `146` +- Scripts: `62` +- Internal script modules: `3` - Secret findings: `0` - Network-capable scripts: `2` +- Network policy covered scripts: `2` +- Network policy missing scripts: `0` +- File-write scripts: `47` +- Permission approvals: `3 / 3` +- Permission approval gaps: `0` +- CLI help smoke checked: `59` +- CLI help smoke failures: `0` - Interactive scripts: `0` -- Package SHA256: `18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad` +- Package hash scope: `source-contract-without-generated-reports` +- Package hash files: `146` +- Package SHA256: `9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe` ## Failures @@ -14,8 +24,7 @@ ## Warnings -- Scripts without argparse/help surface: scripts/render_context_reports.py, scripts/render_social_preview.py, scripts/skill_report_charts.py, scripts/skill_report_metrics.py, scripts/skill_report_model.py -- Network-capable scripts require bounded host policy: scripts/check_update.py, scripts/github_benchmark_scan.py +- None ## Dependency Evidence @@ -23,57 +32,95 @@ - Pinned entries: `1` - Unpinned entries: `0` +## Network Policy + +- Policy file: `security/network_policy.json` +- Present: `True` +- Covered scripts: `2` +- Missing scripts: `none` +- Mismatches: `0` + +## Permission Governance + +- Policy file: `security/permission_policy.json` +- Present: `True` +- Required capabilities: `file_write, network, subprocess` +- Approved capabilities: `file_write, network, subprocess` +- Missing approvals: `none` +- Invalid approvals: `none` +- Expired approvals: `none` + +## CLI Help Smoke + +- Enabled: `True` +- Timeout seconds: `5.0` +- Checked scripts: `59` +- Passed scripts: `59` +- Failed scripts: `none` + ## Script Surface -| Script | Argparse | Main Guard | Input | Network | Subprocess | -| --- | --- | --- | --- | --- | --- | -| scripts/build_confusion_matrix.py | True | True | False | False | False | -| scripts/build_skill_atlas.py | True | True | False | False | False | -| scripts/check_update.py | True | True | False | True | False | -| scripts/ci_test.py | True | True | False | False | True | -| scripts/collect_feedback.py | True | True | False | False | False | -| scripts/context_sizer.py | True | True | False | False | False | -| scripts/create_iteration_snapshot.py | True | True | False | False | False | -| scripts/cross_packager.py | True | True | False | False | False | -| scripts/diff_eval.py | True | True | False | False | False | -| scripts/export_skill_ir.py | True | True | False | False | False | -| scripts/github_benchmark_scan.py | True | True | False | True | False | -| scripts/governance_check.py | True | True | False | False | False | -| scripts/init_skill.py | True | True | False | False | False | -| scripts/judge_blind_eval.py | True | True | False | False | False | -| scripts/lint_skill.py | True | True | False | False | False | -| scripts/optimize_description.py | True | True | False | False | False | -| scripts/promotion_checker.py | True | True | False | False | False | -| scripts/render_artifact_design_profile.py | True | True | False | False | False | -| scripts/render_baseline_compare.py | True | True | False | False | False | -| scripts/render_context_reports.py | False | True | False | False | False | -| scripts/render_description_drift_history.py | True | True | False | False | False | -| scripts/render_eval_dashboard.py | True | True | False | False | True | -| scripts/render_intent_confidence.py | True | True | False | False | False | -| scripts/render_intent_dialogue.py | True | True | False | False | False | -| scripts/render_iteration_directions.py | True | True | False | False | False | -| scripts/render_iteration_ledger.py | True | True | False | False | False | -| scripts/render_output_risk_profile.py | True | True | False | False | False | -| scripts/render_portability_report.py | True | True | False | False | False | -| scripts/render_prompt_quality_profile.py | True | True | False | False | False | -| scripts/render_reference_scan.py | True | True | False | False | False | -| scripts/render_reference_synthesis.py | True | True | False | False | False | -| scripts/render_regression_history.py | True | True | False | False | False | -| scripts/render_review_studio.py | True | True | False | False | False | -| scripts/render_review_viewer.py | True | True | False | False | False | -| scripts/render_skill_overview.py | True | True | False | False | False | -| scripts/render_social_preview.py | False | True | False | False | False | -| scripts/render_system_model.py | True | True | False | False | False | -| scripts/resource_boundary_check.py | True | True | False | False | False | -| scripts/run_conformance_suite.py | True | True | False | False | False | -| scripts/run_description_optimization_suite.py | True | True | False | False | False | -| scripts/run_eval_suite.py | True | True | False | False | True | -| scripts/run_output_eval.py | True | True | False | False | False | -| scripts/skill_report_charts.py | False | False | False | False | False | -| scripts/skill_report_metrics.py | False | False | False | False | False | -| scripts/skill_report_model.py | False | False | False | False | False | -| scripts/sync_local_install.py | True | True | False | False | True | -| scripts/trigger_eval.py | True | True | False | False | False | -| scripts/trust_check.py | True | True | False | False | False | -| scripts/validate_skill.py | True | True | False | False | False | -| scripts/yao.py | True | True | False | False | True | +| Script | Interface | Declared | Argparse | Main Guard | Input | Network | File Write | Subprocess | Reason | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| scripts/adjudicate_output_review.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/build_confusion_matrix.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/build_skill_atlas.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/check_update.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/ci_test.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/collect_feedback.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/compile_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/context_sizer.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/create_iteration_snapshot.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/cross_packager.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/diff_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/export_skill_ir.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/github_benchmark_scan.py | cli | False | True | True | False | True | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/governance_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/init_skill.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/judge_blind_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/lint_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/local_output_eval_runner.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/optimize_description.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/probe_runtime_permissions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/promotion_checker.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/registry_audit.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_adoption_drift_report.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_artifact_design_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_baseline_compare.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_context_reports.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_description_drift_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_eval_dashboard.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_intent_confidence.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_intent_dialogue.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_iteration_directions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_iteration_ledger.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_output_risk_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_portability_report.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_prompt_quality_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_reference_scan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_reference_synthesis.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_regression_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_review_annotations.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_review_studio.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_review_viewer.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_review_waivers.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_skill_overview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_social_preview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_system_model.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/run_conformance_suite.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/run_description_optimization_suite.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/run_eval_suite.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/run_output_eval.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/run_output_execution.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/simulate_install.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/skill_report_charts.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to render inline SVG report charts. | +| scripts/skill_report_metrics.py | internal-module | True | False | False | False | False | False | False | Imported by skill_report_model.py to calculate overview report metrics. | +| scripts/skill_report_model.py | internal-module | True | False | False | False | False | False | False | Imported by render_skill_overview.py to build the v2 report data model. | +| scripts/sync_local_install.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/trigger_eval.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/trust_check.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/upgrade_check.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/validate_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/yao.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 6fa359a..ae003a9 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -8,80 +8,114 @@ Scope: Yao Meta Skill against the user-provided Skill OS 2.0 upgrade plan. Yao Meta Skill is no longer only a Meta Skill factory. The current working tree now has the first verifiable Skill OS foundation: - Skill IR v0 for platform-neutral contract capture. -- Output Eval Lab v0 for with-skill vs baseline assertion grading. +- Output Eval Lab v0 for with-skill vs baseline assertion grading plus local command-runner execution evidence, blind A/B review pack, separate answer key evidence, and reviewer adjudication reporting. - Runtime Conformance v0 for target-consumption checks. -- Trust/Security v0 for secret, dependency, script, trust metadata, and package-integrity checks. +- Trust/Security v0 for secret, dependency, script, bounded network host policy, execution-level CLI help smoke checks, trust metadata, and stable source-contract integrity checks. - Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, and no-route opportunities. - Bilingual Skill Overview v2 that includes these evidence surfaces. -- Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, and release-gate review. -- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, parity checks, and IR provenance. +- Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review. +- Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers. +- Target Compiler v0 so Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts before packaging, including target-specific permission contracts and native behavior contracts. +- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, compiler contract, parity checks, and IR provenance. +- Registry & Distribution v0 for package metadata, checksum, owner, license, compatibility matrix, and audit reporting. +- Package Verification v0 for generated manifest, target adapters, zip archive safety, archive checksum, and registry parity. +- Install Simulation v0 for local extraction, entrypoint loading, interface loading, report presence, and adapter readability. +- Upgrade Check v0 for registry package diffs, semver bump recommendations, breaking-change notes, and release evidence. +- Adoption Drift v0 for local-first metadata-only telemetry, privacy-field blocking, usage-signal aggregation, and iteration candidate generation. +- Review Waivers v0 for human warning acceptance with reviewer, reason, scope, expiry, and blocker-safe policy. +- Governed Permission Gates v0 for reviewer-approved network, file-write, and subprocess capabilities with scope, reason, expiry, evidence, and target-enforcement mapping. +- Runtime Permission Probes v0 for packaged target adapter checks, explicit native-enforcement flags, metadata fallback evidence, and residual permission risks. +- Atlas Scope Policy v0 so examples, evolution snapshots, embedded generated skills, and validator fixtures remain visible in the full portfolio report without polluting release-actionable gates. +- Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision. +- Review Studio now reaches a clean `ready` decision for the root Meta Skill: `13 / 13` gates pass, world-class score `100`, no blockers, no warnings, and no review actions required. -This is still not the final world-class state. Registry, full target-specific compiler transforms, telemetry, and stricter governed trust gates remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit and local output-eval command execution is wired, but deeper provider-native execution transforms, real client telemetry capture, installer integration, provider-backed model-executed output eval, real human adjudication decisions, and native runtime permission enforcement remain open. ## Coverage Matrix | 2.0 Area | Current Evidence | Status | | --- | --- | --- | | Skill IR | `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py` | v0 landed | -| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `reports/output_quality_scorecard.md` | v0 landed | +| Target Compiler | `scripts/compile_skill.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed | +| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_adjudication.md` | v0 landed | | Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | -| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` | v0 landed | +| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed | +| Review Studio Source Refs | `scripts/render_review_studio.py`, `tests/verify_review_studio.py`, `references/review-studio-method.md` | v0 landed | +| Review Annotations | `scripts/render_review_annotations.py`, `reports/review_annotations.md`, `tests/verify_review_annotations.py`, Review Studio annotation panel and blocker decision hook | v0 landed | | Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `reports/skill_atlas.html` | v0 landed | -| Registry & Distribution | IR-sourced packager, no registry audit/package schema yet | partial | -| Telemetry & Drift | Regression history exists, no adoption or activation telemetry yet | partial | -| Compiler from IR | Packager consumes Skill IR for core semantic adapter fields and keeps frontmatter/interface parity checks | v0 landed | +| Registry & Distribution | `registry/*.schema.json`, `scripts/registry_audit.py`, `reports/registry_audit.md`, `registry/packages/yao-meta-skill.json` | v0 landed | +| Package Verification | `scripts/verify_package.py`, `reports/package_verification.md`, `tests/verify_package_verification.py` | v0 landed | +| Install Simulation | `scripts/simulate_install.py`, `reports/install_simulation.md`, `tests/verify_install_simulation.py` | v0 landed | +| Upgrade Check | `scripts/upgrade_check.py`, `reports/upgrade_check.md`, `tests/verify_upgrade_check.py` | v0 landed | +| Telemetry & Drift | `scripts/render_adoption_drift_report.py`, `reports/adoption_drift_report.md`, `tests/verify_adoption_drift.py`, `references/telemetry-drift-method.md` | v0 landed | +| Review Waivers | `scripts/render_review_waivers.py`, `reports/review_waivers.md`, `tests/verify_review_waivers.py`, `references/review-waiver-method.md` | v0 landed | +| Governed Permission Gates | `security/permission_policy.json`, `scripts/trust_check.py`, `scripts/render_review_studio.py`, `tests/verify_trust_check.py`, `tests/verify_review_studio.py` | v0 landed | +| Runtime Permission Probes | `scripts/probe_runtime_permissions.py`, `reports/runtime_permission_probes.md`, `tests/verify_runtime_permission_probes.py`, `tests/verify_review_studio.py` | v0 landed | +| Atlas Scope Policy | `skill_atlas/policy.json`, `scripts/build_skill_atlas.py`, `tests/verify_skill_atlas.py` | v0 landed | +| Compiler from IR | Packager consumes compiled target contracts for compiler provenance, generated files, adapter modes, permissions, preserved semantics, warnings, and unsupported features | v0 landed | +| Target Native Contracts | `scripts/compile_skill.py`, `scripts/cross_packager.py`, `reports/compiled_targets.md`, adapter snapshots, and generated target outputs carry native surface, activation policy, resource/script strategy, permission enforcement, install scope, review artifacts, fallback behavior, and unsupported native features | v0 landed | ## Top Findings -### 1. Compiler path is IR-first v0, but transforms are still shallow +### 1. Target-native behavior contracts are landed, but provider execution is still shallow -The packager now reads Skill IR for core semantic fields and emits provenance, schema version, job-to-be-done, semantic contract counts, governance, risk, targets, and semantic parity in each adapter. +`scripts/compile_skill.py` now reads Skill IR and emits target-specific contracts for OpenAI, Claude, generic, and Agent Skills compatible outputs. The packager embeds `compiler`, `compiled_contract`, `permission_contract`, `target_permission_contract`, `target_native_contract`, `target_transform`, warnings, and unsupported-feature notes in each adapter. The native contract makes the target surface, activation policy, resource strategy, script strategy, permission enforcement mode, install scope, review artifacts, fallback behavior, and unsupported native features reviewable instead of implicit. -Next move: add `scripts/compile_skill.py` or split the packager into target-specific transforms so OpenAI, Claude, Agent Skills, VS Code/Copilot, and generic packages can preserve runtime semantics beyond metadata. +Next move: deepen provider-native execution transforms so OpenAI, Claude, VS Code/Copilot, and generic packages can express and verify behavior through real client or installer capabilities, not only auditable contracts and fallback notes. -### 2. Output eval now meets the governed v0 minimum, but is still static +### 2. Output eval now has command-runner evidence, but provider model execution is still open -The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. The next gap is that they are still static text comparisons rather than model-executed holdout runs with timing, tokens, and reviewer adjudication. +The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. Each run emits assertion grading, execution provenance, a blind A/B review pack that hides baseline vs with-skill labels, a separate answer key, and an adjudication report from reviewer choices. The current root execution report now runs all `10` variants through `scripts/local_output_eval_runner.py`, so timing and command-runner behavior are observed instead of relying only on static recorded fixtures. It still intentionally reports `0` provider-backed model-executed runs and `10` estimated token counts, because the local runner verifies the harness contract without pretending to be a model provider. The current root adjudication report intentionally records `0 / 5` human judgments pending instead of pretending human review happened. The next gap is provider-backed model-executed holdout runs with real provider timing/tokens and real reviewer decision records. -Next move: add model-executed output eval runs, blind A/B comparison, and one real multi-file fixture. +Next move: add provider-backed model execution for holdout cases and one real multi-file fixture with actual reviewer decisions. -### 3. Review Studio is unified, but needs reviewer actions +### 3. Review Studio is unified and now has reviewer actions plus annotations -The new Review Studio page aggregates intent, trigger, output, context, runtime, trust, atlas, and release gates. It now exposes current warnings directly: low generic intent-confidence context, trust-script warnings, and portfolio-level Atlas gaps. +The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. After adding root intent context, Atlas scope policy, metadata-only telemetry evidence, runtime permission probe evidence, action source refs, and an empty annotation ledger, the current root report is clean: decision `ready`, score `100`, `13` gates, `0` gate blockers, `0` gate warnings, `0` actions, and `0` open annotation blockers. -Next move: add reviewer annotations, waived-risk records, and links from each warning to the exact source fix. +Next move: add richer source-line anchors inside generated reports and record real reviewer annotations during the next human review pass. -### 4. Multi-skill operation now has v0 coverage, but no telemetry +### 4. Multi-skill operation now has v0 coverage, but needs real telemetry -The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, and no-route opportunities. It is still static analysis and does not yet include adoption or activation telemetry. +The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, and no-route opportunities. It now also supports `skill_atlas/policy.json` so release gates distinguish actionable library skills from examples and fixtures while keeping full visibility. Adoption Drift v0 can record metadata-only local events, block raw prompt/output fields, summarize missed-trigger, bad-output, script-error, and review-overdue signals, and feed next iteration candidates into Review Studio. It still needs real client-side capture instead of manual CLI event recording. -Next move: connect telemetry and failure history so Atlas can rank stale or conflicting skills by real usage impact. +Next move: connect client activations and failure history so Atlas can rank stale or conflicting skills by real usage impact. ### 5. Trust report is structural, not full security review -Trust v0 blocks obvious secrets and remote inline execution, but it does not yet execute script `--help`, inspect package archive hashes, or enforce per-target permissions. +Trust v0 blocks obvious secrets and remote inline execution, and now makes checksum scope explicit: `package_sha256` is a stable source-contract digest that excludes generated reports, packages, and raw local telemetry; archive integrity is carried by Package Verification and Registry as `archive_sha256`. Registry v0 carries both checksums into distributable metadata. Package Verification v0 now checks generated manifests, target adapters, archive path safety, required source entries, and registry parity. Install Simulation v0 now extracts the generated archive into a temporary local skill root and verifies entrypoint, manifest, interface, reports, and adapters. Upgrade Check v0 now compares package baselines, recommends semver bumps, and blocks breaking changes without an adequate declared version. Trust Report now distinguishes CLI scripts from declared internal modules, so help-surface warnings focus on executable entrypoints. Network-capable scripts are now covered by `security/network_policy.json`, with `allowed_hosts` checked against HTTPS URL literals. Trust Report also executes `python3 scripts/name.py --help` for CLI scripts with `argparse`, recording pass/fail smoke evidence without executing scripts that lack a help surface. Target adapters now carry `permission_contract` and `target_permission_contract` fields, including network, file-write, subprocess, and interactive capability counts. Governed permission gates now require reviewer-approved scope, reason, expiry, evidence, and target-enforcement mapping in `security/permission_policy.json`. Runtime Permission Probes now verify that packaged adapters expose the target permission contract and explicitly report native-enforcement limits. The remaining trust gap is native runtime permission enforcement in real clients/installers. -Next move: add stricter governed-mode gates and package hash verification after registry format lands. +Next move: add real client or installer permission enforcement integration. ## Current Gate Evidence | Gate | Current Result | | --- | --- | -| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, and boundary coverage | +| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs, `10` estimated token counts, `5` blind A/B review pairs, and `0 / 5` reviewer decisions pending | | Runtime Conformance | `5 / 5` targets passing | -| Trust | `0` secret findings, `1` pinned dependency file, `2` network-capable scripts flagged as warnings | -| Skill Atlas | local workspace scan generated catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview | -| Review Studio | `8` gates, `0` blockers, `3` warnings, world-class score `87/100` | -| IR-first Packaging | `openai`, `claude`, and `generic` adapters include IR provenance and semantic parity checks | -| Context Budget | initial load `910/1000`, under the production budget | -| CI | `make ci-test` passed after adding v0 gates | +| Target Compiler | compiled target contracts generated for OpenAI, Claude, generic, and Agent Skills compatible outputs, including target permission contracts and target-native behavior contracts | +| Trust | `0` secret findings, `1` pinned dependency file, `3` declared internal modules, `2 / 2` network-capable scripts covered by bounded host policy, `59 / 59` CLI help smoke checks passing across `62` scripts, source-contract hash scope explicit | +| Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | +| Runtime Permission Probes | `3 / 3` target adapters probed, `0` native-enforcement adapters, `3` explicit metadata fallbacks, `3` residual risks retained for reviewer visibility | +| Skill Atlas | `12` scanned skills, `1` actionable root skill, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `24` scoped non-actionable issue signals retained for visibility | +| Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | +| Package Verification | `3 / 3` target adapters present, archive verified, `483` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `483` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `3` adapters readable, `0` failures, `0` warnings | +| Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | +| Adoption Drift | `1` metadata-only activation event, adoption `100`, risk band `low`; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | +| Review Waivers | ledger generated; current release has no warning gates to waive, blockers remain non-waivable in v0 | +| Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | +| Review Studio | decision `ready`, world-class score `100`, `13` gates, `0` blockers, `0` warnings, `0` review actions, `0` open annotation blockers | +| IR-first Packaging | `openai`, `claude`, and `generic` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, and semantic parity checks | +| Context Budget | initial load `987/1000`, under the production budget | +| CI | `make ci-test` target count is now `56` after adding Review Annotations | ## Next Highest-Leverage Moves -1. Split IR-first packaging into target-specific compiler transforms. -2. Add reviewer annotation and waived-risk records to Review Studio. -3. Expand Output Eval Lab from static cases to model-executed holdout and blind A/B cases. -4. Add registry package schema, package hash, and upgrade audit. -5. Connect Skill Atlas with telemetry and drift history. +1. Deepen target-native behavior contracts into provider-native execution and installer integrations. +2. Add native client or installer enforcement for approved high-permission capabilities. +3. Expand Output Eval Lab from local command-runner evidence to provider-backed model holdout and real reviewer-adjudicated cases. +4. Add real reviewer annotation records during the next human review pass. +5. Connect Skill Atlas with real client telemetry and drift history. diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 6864237..8d36b95 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -342,6 +342,9 @@ gap: 16px; align-items: stretch; } + .quality-panels { + margin-top: 16px; + } .metrics-stack { display: grid; gap: 18px; @@ -606,7 +609,7 @@ -
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
上下文成本42入口约 346 个词/字，references 约 11313 个词/字。入口约 346 个词/字，references 约 11313 个词/字。
+
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
@@ -618,7 +621,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html
+
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
交付流程输入材料Skill 包体可复用能力
交付流程把用户输入、生成的包体和可复用能力放在一条线上。
@@ -638,10 +641,10 @@

指标判读Reading

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 346 个词/字。SKILL.md 约 346 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 346 个词/字，references 约 11313 个词/字。入口约 346 个词/字，references 约 11313 个词/字。
+
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • intent-dialogue.json 已存在。intent-dialogue.json 已存在。
可维护性100
  • SKILL.md 约 346 个词/字。SKILL.md 约 346 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 346 个词/字，references 约 11313 个词/字。入口约 346 个词/字，references 约 11313 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
@@ -654,10 +657,10 @@

判断这个 Skill 在能力地图中的位置和复用范围。Places this skill on a capability map and clarifies reuse scope.

-
能力矩阵执行确定性知识密度Dialogue interaction
能力矩阵说明这个 Skill 更偏知识密集还是执行确定。
+
能力矩阵执行确定性知识密度Execution operation
能力矩阵说明这个 Skill 更偏知识密集还是执行确定。

画像摘要Profile

-
  • 能力类型：Dialogue interaction能力类型：Dialogue interaction
  • 成熟度：governed成熟度：governed
  • 触发强度：手动触发 + description 路由触发强度：手动触发 + description 路由
  • 复用范围：跨平台复用范围：跨平台
+
  • 能力类型：Execution operation能力类型：Execution operation
  • 成熟度：governed成熟度：governed
  • 触发强度：手动触发 + description 路由触发强度：手动触发 + description 路由
  • 复用范围：跨平台复用范围：跨平台
@@ -696,11 +699,11 @@

触发描述Trigger

该 Skill 的触发描述来自 SKILL.md frontmatter；默认中文报告先呈现能力边界，原始英文描述可切换到英文查看。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

输入材料Inputs

- +

输出结果Outputs

-
  • 可路由的 SKILL.md可路由的 SKILL.md
  • agents/interface.yaml 元数据agents/interface.yaml 元数据
  • 必要的 references、scripts、evals、reports 证据必要的 references、scripts、evals、reports 证据
  • 结构化 Skill 目录，共 8 类关键资产。结构化 Skill 目录，共 8 类关键资产。
+
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.
  • 结构化 Skill 目录，共 8 类关键资产。结构化 Skill 目录，共 8 类关键资产。

不应触发Should Not Trigger

  • 只需要一次性回答、没有复用价值的临时请求。只需要一次性回答、没有复用价值的临时请求。
  • 要求直接执行相邻任务，而不是沉淀或使用这个 Skill。要求直接执行相邻任务，而不是沉淀或使用这个 Skill。
  • 缺少必要事实且用户不允许澄清的场景。缺少必要事实且用户不允许澄清的场景。
@@ -717,8 +720,28 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。纳入下一轮修复Fix next
+
+
+

执行证据Execution Evidence

+
  • 变体运行：10变体运行：10
  • 模型执行：0模型执行：0
  • 记录样本：0记录样本：0
  • Token 估算：10Token 估算：10
+
+
+

盲评审定Blind Adjudication

+
  • 评审进度：0 / 5评审进度：0 / 5
  • 待评审：5待评审：5
  • 一致率：暂无一致率：暂无
  • 非法决策：0非法决策：0
+
+
+
+
+

评审原则Review Rule

+
  • 先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。Record the reviewer's A/B choice before opening the answer key and calculating agreement.
  • 缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.
+
+
+

运行原则Run Rule

+
  • recorded fixture 只能证明可复现样本，不等同于模型执行。A recorded fixture proves reproducible samples only; it is not model execution.
  • 只有 provider runner 返回 model metadata 时才计入 model-executed。Only provider runners that return model metadata count as model-executed.
+
+
@@ -734,7 +757,7 @@
- +
风险Risk信号Signal应对Response
误触发风险误触发风险frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
输出漂移风险输出漂移风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
证据不足风险证据不足风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
包体膨胀风险包体膨胀风险SKILL.md 约 346 个词/字。SKILL.md 约 346 个词/字。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
跨平台迁移风险跨平台迁移风险agents/interface.yaml 已存在。agents/interface.yaml 已存在。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
误触发风险误触发风险frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
输出漂移风险输出漂移风险已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
证据不足风险证据不足风险已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
包体膨胀风险包体膨胀风险SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
跨平台迁移风险跨平台迁移风险agents/interface.yaml 已存在。agents/interface.yaml 已存在。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
@@ -748,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布255项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布296项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 47b39ad..deb9e89 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -12,10 +12,23 @@ "SKILL.md", "agents/interface.yaml", "reports/skill-ir.json", + "reports/compiled_targets.md", "reports/output_quality_scorecard.md", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.md", "reports/conformance_matrix.md", "reports/security_trust_report.md", + "reports/runtime_permission_probes.md", "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/review_annotations.md", "reports/review-studio.html", "reports/skill-overview.html" ], @@ -52,17 +65,17 @@ "label": "证据充分", "score": 100, "reasons": [ - "已生成 13 / 13 类报告证据。", + "已生成 20 / 20 类报告证据。", "skill-ir.json 已存在。", - "intent-dialogue.json 已存在。", - "intent-confidence.json 已存在。" + "compiled_targets.json 已存在。", + "intent-dialogue.json 已存在。" ] }, "maintainability_score": { "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 346 个词/字。", + "SKILL.md 约 365 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -83,7 +96,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 346 个词/字，references 约 11313 个词/字。", + "入口约 365 个词/字，references 约 14247 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -91,7 +104,7 @@ }, "capability_profile": { "archetype": "governed", - "task_family": "Dialogue interaction", + "task_family": "Execution operation", "maturity": "governed", "trigger_strength": "手动触发 + description 路由", "reuse_scope": "跨平台", @@ -104,7 +117,7 @@ "nodes": [ { "title": "意图澄清", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." }, { "title": "边界路由", @@ -124,7 +137,7 @@ }, { "title": "杠杆升级", - "body": "Ask one focused question and update intent context before adding assets." + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." } ], "layers": [ @@ -143,13 +156,13 @@ "shell": "bash" }, "inputs": [ - "用户提供的工作流、提示词、文档、记录或散乱笔记", - "期望沉淀的复用场景、排除项、约束和质量标准" + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" ], "outputs": [ - "可路由的 SKILL.md", - "agents/interface.yaml 元数据", - "必要的 references、scripts、evals、reports 证据", + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "结构化 Skill 目录，共 8 类关键资产。" ], "should_trigger": [ @@ -181,13 +194,13 @@ "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", - "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", - "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", - "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。" + "上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -195,7 +208,7 @@ "每次升级后重新生成报告并检查分数原因。" ], "artifact_design": { - "design_system": "review studio", + "design_system": "metric editorial", "highlights": [ "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", "Name the working directory and required inputs for commands.", @@ -204,24 +217,24 @@ ] }, "prompt_quality": { - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "highlights": [ - "Primary prompt task family: Dialogue interaction.", - "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", - "Completeness: 80/100.", - "Clarity: 80/100." + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." ] }, "system_model": { "stability": { - "score": 86, - "band": "stable-first-pass" + "score": 100, + "band": "system-ready" }, "highlights": [ - "Stability: stable-first-pass (86/100).", - "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", - "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." ] } }, @@ -238,21 +251,21 @@ "name": "输出漂移风险", "impact": 2, "probability": 1, - "signal": "已生成 13 / 13 类报告证据。", + "signal": "已生成 20 / 20 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "证据不足风险", "impact": 3, "probability": 1, - "signal": "已生成 13 / 13 类报告证据。", + "signal": "已生成 20 / 20 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 346 个词/字。", + "signal": "SKILL.md 约 365 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -414,13 +427,13 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 29 + "file_count": 32 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 50 + "file_count": 62 }, { "path": "evals", @@ -432,10 +445,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 144 + "file_count": 170 } ], - "file_count": 255, + "file_count": 296, "folder_count": 4, "distribution": [ { @@ -456,11 +469,11 @@ }, { "label": "references", - "value": 29 + "value": 32 }, { "label": "scripts", - "value": 50 + "value": 62 }, { "label": "evals", @@ -468,7 +481,7 @@ }, { "label": "reports", - "value": 144 + "value": 170 } ] }, @@ -583,13 +596,13 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 29 + "file_count": 32 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 50 + "file_count": 62 }, { "path": "evals", @@ -601,16 +614,16 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 144 + "file_count": 170 } ], "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。", "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", - "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", - "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", - "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "scenarios": [ "把重复流程沉淀为可复用的 agent skill。", @@ -626,20 +639,20 @@ }, "io_contract": { "inputs": [ - "用户提供的工作流、提示词、文档、记录或散乱笔记", - "期望沉淀的复用场景、排除项、约束和质量标准" + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" ], "outputs": [ - "可路由的 SKILL.md", - "agents/interface.yaml 元数据", - "必要的 references、scripts、evals、reports 证据", + "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "结构化 Skill 目录，共 8 类关键资产。" ] }, "principles": [ { "title": "意图澄清", - "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + "body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse." }, { "title": "边界路由", @@ -659,7 +672,7 @@ }, { "title": "杠杆升级", - "body": "Ask one focused question and update intent context before adding assets." + "body": "Name the recurring job, expected input, output, and strongest non-goal in compact language." } ], "roadmap": [ @@ -707,6 +720,81 @@ "trigger_samples": 8, "output_eval_cases": 3 }, + "compiled_targets": { + "ok": true, + "schema_version": "1.0", + "summary": { + "target_count": 4, + "pass_count": 4, + "warn_count": 0, + "block_count": 0, + "failure_count": 0, + "warning_count": 0 + }, + "targets": [ + { + "target": "openai", + "status": "pass", + "adapter_mode": "metadata-adapter", + "degradation_strategy": "metadata-adapter", + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "permission_enforcement": "metadata-only", + "generated_files": [ + "targets/openai/adapter.json", + "targets/openai/agents/openai.yaml" + ], + "unsupported_features": [ + "client-native script permission prompts are represented as permission contract metadata" + ], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "adapter_mode": "neutral-source-plus-adapter", + "degradation_strategy": "neutral-source-plus-adapter", + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "permission_enforcement": "metadata-fallback", + "generated_files": [ + "targets/claude/adapter.json", + "targets/claude/README.md" + ], + "unsupported_features": [ + "vendor-native metadata fields are carried as adapter JSON and README notes" + ], + "warnings": [] + }, + { + "target": "generic", + "status": "pass", + "adapter_mode": "agent-skills-compatible", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills compatible neutral package", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "targets/generic/adapter.json" + ], + "unsupported_features": [], + "warnings": [] + }, + { + "target": "agent-skills-compatible", + "status": "pass", + "adapter_mode": "neutral-agent-skills-source", + "degradation_strategy": "neutral-source", + "native_surface": "Agent Skills standard source tree", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "generated_files": [ + "SKILL.md", + "agents/interface.yaml" + ], + "unsupported_features": [], + "warnings": [] + } + ], + "failures": [], + "warnings": [] + }, "output_quality": { "case_count": 5, "file_backed_case_count": 1, @@ -717,39 +805,417 @@ "delta": 100.0, "regression_count": 0, "gate_pass": true, - "failure_taxonomy": [] + "failure_taxonomy": [], + "blind_pair_count": 5 + }, + "output_execution": { + "ok": true, + "summary": { + "case_count": 5, + "variant_run_count": 10, + "command_executed_count": 10, + "model_executed_count": 0, + "recorded_fixture_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "token_estimated_count": 10, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "failure_count": 0, + "gate_pass": true + }, + "runner": { + "mode": "command", + "command": [ + "python3", + "scripts/local_output_eval_runner.py" + ], + "timeout_seconds": 30.0 + }, + "failures": [] + }, + "output_blind_review": { + "summary": { + "pair_count": 5, + "answer_key_separate": true, + "with_skill_hidden_count": 5 + }, + "seed": "yao-output-eval-blind-v1", + "pair_count": 5, + "answer_key_separate": true + }, + "output_review_adjudication": { + "ok": true, + "summary": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "agreement_count": 0, + "disagreement_count": 0, + "invalid_decision_count": 0, + "agreement_rate": null, + "needs_review": true, + "failure_count": 0 + }, + "reviewer": "", + "reviewed_at": "", + "failures": [] }, "runtime_conformance": { "target_count": 5, "pass_count": 5, "fail_count": 0 }, + "runtime_permissions": { + "ok": true, + "summary": { + "target_count": 3, + "pass_count": 3, + "fail_count": 0, + "native_enforcement_count": 0, + "metadata_fallback_count": 3, + "residual_risk_count": 3, + "required_capability_count": 3, + "failure_count": 0 + }, + "expected_capabilities": [ + "file_write", + "network", + "subprocess" + ], + "targets": [ + { + "target": "openai", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "claude", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + }, + { + "target": "generic", + "status": "pass", + "assurance": "metadata-fallback-explicit", + "native_enforcement": false, + "metadata_fallback_explicit": true, + "residual_risks": [ + "Client-native permission enforcement is not provided by this target; installer or operator must honor metadata." + ] + } + ], + "failures": [] + }, "trust_security": { - "scanned_files": 127, - "script_count": 50, + "scanned_files": 146, + "script_count": 62, + "internal_module_count": 3, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], "network_script_count": 2, + "network_policy_covered_count": 2, + "network_policy_missing_count": 0, + "file_write_script_count": 47, + "permission_required_count": 3, + "permission_approved_count": 3, + "permission_missing_count": 0, + "permission_invalid_count": 0, + "permission_expired_count": 0, + "help_smoke_checked_count": 59, + "help_smoke_failed_count": 0, "interactive_script_count": 0, - "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" + "package_hash_scope": "source-contract-without-generated-reports", + "package_hash_file_count": 146, + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" }, "skill_atlas": { "skill_count": 12, + "actionable_skill_count": 1, "route_collision_count": 5, + "actionable_route_collision_count": 0, "owner_gap_count": 9, + "actionable_owner_gap_count": 0, "stale_count": 10, + "actionable_stale_count": 0, "shared_resource_count": 0, - "no_route_opportunity_count": 0 + "no_route_opportunity_count": 0, + "non_actionable_issue_count": 24 + }, + "registry_distribution": { + "ok": true, + "package": { + "schema_version": "2.0", + "name": "yao-meta-skill", + "version": "1.1.0", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "maturity": "governed", + "owner": "Yao Team", + "review_cadence": "quarterly", + "trust_level": "local", + "license": "MIT", + "checksums": { + "package_sha256": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe", + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + "compatibility": { + "openai": "pass", + "claude": "pass", + "agent-skills": "pass", + "vscode": "pass", + "generic": "pass", + "agent-skills-compatible": "pass" + }, + "source": { + "skill_dir": ".", + "skill_ir": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0", + "canonical_metadata": "agents/interface.yaml" + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "package_metadata": "registry/packages/yao-meta-skill.json" + }, + "distribution": { + "archive_verified": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "package_verification": "reports/package_verification.json", + "install_simulated": true, + "install_simulation": "reports/install_simulation.json" + }, + "generated_at": "2026-06-13" + }, + "failures": [], + "warnings": [] + }, + "package_verification": { + "ok": true, + "summary": { + "target_count": 3, + "adapter_count": 3, + "archive_present": true, + "archive_sha256": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06", + "archive_entry_count": 483, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "install_simulation": { + "ok": true, + "summary": { + "archive_present": true, + "archive_entry_count": 483, + "archive_extracted": true, + "entrypoint_loaded": true, + "manifest_loaded": true, + "interface_loaded": true, + "adapter_count": 3, + "install_root_is_temp": false, + "failure_count": 0, + "warning_count": 0 + }, + "failures": [], + "warnings": [] + }, + "upgrade_check": { + "ok": true, + "summary": { + "declared_bump": "minor", + "recommended_bump": "minor", + "breaking_change_count": 0, + "added_target_count": 1, + "metadata_change_count": 1, + "checksum_change_count": 2, + "failure_count": 0, + "warning_count": 0 + }, + "upgrade_diff": { + "name_changed": false, + "added_targets": [ + "agent-skills-compatible" + ], + "removed_targets": [], + "compatibility_changes": [ + { + "target": "agent-skills", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "agent-skills-compatible", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "vscode", + "from": "missing", + "to": "pass", + "direction": "improved" + } + ], + "metadata_changes": [ + { + "field": "description", + "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + } + ], + "checksum_changes": [ + { + "field": "archive_sha256", + "from": "", + "to": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + { + "field": "package_sha256", + "from": "0000000000000000000000000000000000000000000000000000000000000000", + "to": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" + } + ] + }, + "release_notes": [ + "Recommended version bump: minor.", + "Added targets: agent-skills-compatible.", + "Package or archive checksum changed; reviewers should verify package artifacts before release.", + "Package verification evidence: reports/package_verification.md." + ], + "failures": [], + "warnings": [] + }, + "adoption_drift": { + "ok": true, + "summary": { + "event_count": 1, + "activation_count": 1, + "accepted_count": 1, + "edited_count": 0, + "rejected_count": 0, + "missed_count": 0, + "failed_count": 0, + "adoption_rate": 100.0, + "missed_trigger_count": 0, + "wrong_trigger_count": 0, + "bad_output_count": 0, + "script_error_count": 0, + "missing_resource_count": 0, + "review_overdue_count": 0, + "risk_band": "low", + "event_types": { + "skill_activation": 1 + }, + "failure_types": {} + }, + "next_iteration_candidates": [], + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": false, + "raw_event_log_packaged": false, + "blocked_fields": [ + "content", + "input", + "inputs", + "message", + "messages", + "note", + "output", + "outputs", + "prompt", + "raw", + "text", + "transcript" + ] + }, + "failures": [] + }, + "review_waivers": { + "ok": true, + "summary": { + "waiver_count": 0, + "active_count": 0, + "expired_count": 0, + "invalid_count": 0, + "covered_gate_count": 0, + "covered_gate_keys": [] + }, + "policy": { + "blocker_waivers_allowed": false, + "minimum_reason_chars": 20, + "expires_required": true, + "known_gate_keys": [ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "permission-gates", + "permission-runtime", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report" + ] + }, + "failures": [], + "warnings": [] + }, + "review_annotations": { + "ok": true, + "summary": { + "annotation_count": 0, + "open_count": 0, + "resolved_count": 0, + "deferred_count": 0, + "open_blocker_count": 0, + "open_warning_count": 0, + "linked_gate_count": 0, + "target_missing_count": 0, + "failure_count": 0 + }, + "annotations": [], + "failures": [] }, "synthesis_highlights": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed.", "Borrow the discipline of defining what the skill should not own before growing the package." ], "artifact_design": { - "design_system": "review studio", + "design_system": "metric editorial", "highlights": [ "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", "Name the working directory and required inputs for commands.", @@ -758,24 +1224,24 @@ ] }, "prompt_quality": { - "overall_quality_score": 82.0, + "overall_quality_score": 89.0, "highlights": [ - "Primary prompt task family: Dialogue interaction.", - "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", - "Completeness: 80/100.", - "Clarity: 80/100." + "Primary prompt task family: Execution operation.", + "Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints", + "Completeness: 100/100.", + "Clarity: 85/100." ] }, "system_model": { "stability": { - "score": 86, - "band": "stable-first-pass" + "score": 100, + "band": "system-ready" }, "highlights": [ - "Stability: stable-first-pass (86/100).", - "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", - "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", - "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + "Stability: system-ready (100/100).", + "Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.", + "Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes." ] }, "metadata": { diff --git a/reports/skill_atlas.html b/reports/skill_atlas.html index a3ebd8b..0c14b4a 100644 --- a/reports/skill_atlas.html +++ b/reports/skill_atlas.html @@ -25,20 +25,25 @@

Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

Skills12
-
Route Collisions5
-
Owner Gaps9
-
Stale Skills10
+
Actionable1
+
Route Collisions0
+
Owner Gaps0
+
Stale Skills0
No-Route Opportunities0
-

Top Issues

-
  • frontend-review collision
  • frontend-review collision
  • frontend-review collision
  • broken-yaml-skill collision
  • frontend-review duplicate-name
  • frontend-review owner, review_cadence, maturity
  • frontend-review owner, review_cadence, maturity
  • frontend-review owner, review_cadence, maturity
  • note-cleanup owner, review_cadence, maturity
  • frontend-review owner, review_cadence, maturity
  • geo-ranking-article-generator owner, review_cadence, maturity
  • invalid-governance-skill owner, review_cadence
  • broken-yaml-skill owner, review_cadence, maturity
  • broken-skill owner, review_cadence, maturity
  • frontend-review missing updated_at
  • frontend-review missing updated_at
  • frontend-review missing updated_at
  • incident-command-governor review overdue by cadence monthly
  • note-cleanup missing updated_at
  • frontend-review missing updated_at
  • geo-ranking-article-generator missing updated_at
  • invalid-governance-skill missing updated_at
  • broken-yaml-skill missing updated_at
  • broken-skill missing updated_at
+

Actionable Issues

+
  • No blocking portfolio issues detected.
+
+
+

Full Portfolio Counts

+

All scanned skills remain visible: 5 total route collisions, 9 total owner gaps, and 10 total stale signals.

Catalog

路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
- - + +
NamePathOwnerMaturityReview
yao-meta-skill.Yao Teamgovernedquarterly
release-orchestratorexamples/complex-release-orchestrator/generated-skillYao Teamproductionquarterly
frontend-reviewexamples/evolution-frontend-review/finalmissingunknownmissing
frontend-reviewexamples/evolution-frontend-review/v0missingunknownmissing
frontend-reviewexamples/evolution-frontend-review/v1missingunknownmissing
incident-command-governorexamples/governed-incident-command/generated-skillIncident Operations Teamgovernedmonthly
note-cleanupexamples/simple-note-cleanup/generated-skillmissingunknownmissing
frontend-reviewexamples/team-frontend-review/generated-skillmissingunknownmissing
geo-ranking-article-generatorgeo-ranking-article-generatormissingunknownmissing
invalid-governance-skilltests/fixtures/governance_invalid_manifestmissinggovernedmissing
broken-yaml-skilltests/fixtures/package_invalid_yamlmissingunknownmissing
broken-skilltests/fixtures/package_missing_interface_fieldmissingunknownmissing
NamePathOwnerMaturityReviewScope
yao-meta-skill.Yao Teamgovernedquarterlyrelease
release-orchestratorexamples/complex-release-orchestrator/generated-skillYao Teamproductionquarterlyexample
frontend-reviewexamples/evolution-frontend-review/finalmissingunknownmissingexample
frontend-reviewexamples/evolution-frontend-review/v0missingunknownmissingexample
frontend-reviewexamples/evolution-frontend-review/v1missingunknownmissingexample
incident-command-governorexamples/governed-incident-command/generated-skillIncident Operations Teamgovernedmonthlyexample
note-cleanupexamples/simple-note-cleanup/generated-skillmissingunknownmissingexample
frontend-reviewexamples/team-frontend-review/generated-skillmissingunknownmissingexample
geo-ranking-article-generatorgeo-ranking-article-generatormissingunknownmissingembedded-generated-skill
invalid-governance-skilltests/fixtures/governance_invalid_manifestmissinggovernedmissingtest-fixture
broken-yaml-skilltests/fixtures/package_invalid_yamlmissingunknownmissingtest-fixture
broken-skilltests/fixtures/package_missing_interface_fieldmissingunknownmissingtest-fixture
diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index c6d1df2..3ef71e2 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -3,11 +3,41 @@ "workspace_root": ".", "summary": { "skill_count": 12, + "actionable_skill_count": 1, "route_collision_count": 5, + "actionable_route_collision_count": 0, "owner_gap_count": 9, + "actionable_owner_gap_count": 0, "stale_count": 10, + "actionable_stale_count": 0, "shared_resource_count": 0, - "no_route_opportunity_count": 0 + "no_route_opportunity_count": 0, + "non_actionable_issue_count": 24 + }, + "scope_policy": { + "present": true, + "path": "skill_atlas/policy.json", + "schema_version": "1.0", + "rules": [ + { + "path_prefix": "examples/", + "scope": "example", + "actionable": false, + "reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + }, + { + "path_prefix": "tests/fixtures/", + "scope": "test-fixture", + "actionable": false, + "reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + }, + { + "path_prefix": "geo-ranking-article-generator", + "scope": "embedded-generated-skill", + "actionable": false, + "reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + } + ] }, "catalog": { "workspace_root": ".", @@ -30,11 +60,13 @@ "agent-skills-compatible" ], "resources": [ + "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", + "scripts/compile_skill.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", @@ -45,8 +77,12 @@ "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", "scripts/render_context_reports.py", @@ -62,8 +98,10 @@ "scripts/render_reference_scan.py", "scripts/render_reference_synthesis.py", "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", "scripts/render_review_studio.py", "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", @@ -72,16 +110,21 @@ "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", "scripts/skill_report_charts.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/upgrade_check.py", "scripts/validate_skill.py", + "scripts/verify_package.py", "scripts/yao.py", "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", "references/governance.md", @@ -102,12 +145,14 @@ "references/regression-cause-taxonomy.md", "references/resource-boundaries.md", "references/review-studio-method.md", + "references/review-waiver-method.md", "references/runtime-conformance-method.md", "references/skill-archetypes.md", "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", "references/trust-security-method.md", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" @@ -131,7 +176,10 @@ "team", "transcripts", "workflows" - ] + ], + "atlas_scope": "release", + "actionable": true, + "scope_reason": "default release-actionable skill" }, { "name": "release-orchestrator", @@ -186,7 +234,10 @@ "stakeholder", "theory", "workflow" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -222,7 +273,10 @@ "states", "ui", "ux" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -242,7 +296,10 @@ "frontend", "merge", "review" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -277,7 +334,10 @@ "security", "states", "ui" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "incident-command-governor", @@ -315,7 +375,10 @@ "run", "severity", "standardize" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "note-cleanup", @@ -349,7 +412,10 @@ "separate", "structured", "summaries" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -386,7 +452,10 @@ "ui", "unsafe", "ux" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "geo-ranking-article-generator", @@ -443,7 +512,10 @@ "write", "writing", "生成式引擎优化" - ] + ], + "atlas_scope": "embedded-generated-skill", + "actionable": false, + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." }, { "name": "invalid-governance-skill", @@ -462,7 +534,10 @@ "governance", "test", "validation" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." }, { "name": "broken-yaml-skill", @@ -483,7 +558,10 @@ "packager", "testing", "yaml" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." }, { "name": "broken-skill", @@ -503,16 +581,24 @@ "fixture", "packager", "testing" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." } ], "summary": { "skill_count": 12, + "actionable_skill_count": 1, "route_collision_count": 5, + "actionable_route_collision_count": 0, "owner_gap_count": 9, + "actionable_owner_gap_count": 0, "stale_count": 10, + "actionable_stale_count": 0, "shared_resource_count": 0, - "no_route_opportunity_count": 0 + "no_route_opportunity_count": 0, + "non_actionable_issue_count": 24 } }, "route_collisions": [ @@ -522,7 +608,10 @@ "path_a": "examples/evolution-frontend-review/final", "path_b": "examples/evolution-frontend-review/v1", "score": 0.708, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "example", + "scope_b": "example" }, { "skill_a": "frontend-review", @@ -530,7 +619,10 @@ "path_a": "examples/evolution-frontend-review/final", "path_b": "examples/team-frontend-review/generated-skill", "score": 0.64, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "example", + "scope_b": "example" }, { "skill_a": "frontend-review", @@ -538,7 +630,10 @@ "path_a": "examples/evolution-frontend-review/v1", "path_b": "examples/team-frontend-review/generated-skill", "score": 0.538, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "example", + "scope_b": "example" }, { "skill_a": "broken-yaml-skill", @@ -546,7 +641,10 @@ "path_a": "tests/fixtures/package_invalid_yaml", "path_b": "tests/fixtures/package_missing_interface_field", "score": 0.833, - "status": "collision" + "status": "collision", + "actionable": false, + "scope_a": "test-fixture", + "scope_b": "test-fixture" }, { "skill_a": "frontend-review", @@ -554,9 +652,13 @@ "path_a": "examples/evolution-frontend-review/final", "path_b": "examples/evolution-frontend-review/v0", "score": 1.0, - "status": "duplicate-name" + "status": "duplicate-name", + "actionable": false, + "scope_a": "example", + "scope_b": "example" } ], + "actionable_route_collisions": [], "dependency_graph": { "nodes": [ { @@ -614,56 +716,77 @@ { "name": "frontend-review", "path": "examples/evolution-frontend-review/final", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/evolution-frontend-review/v0", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/evolution-frontend-review/v1", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", "age_days": 74, - "allowed_days": 31 + "allowed_days": 31, + "actionable": false, + "scope": "example" }, { "name": "note-cleanup", "path": "examples/simple-note-cleanup/generated-skill", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/team-frontend-review/generated-skill", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "geo-ranking-article-generator", "path": "geo-ranking-article-generator", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "embedded-generated-skill" }, { "name": "invalid-governance-skill", "path": "tests/fixtures/governance_invalid_manifest", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-yaml-skill", "path": "tests/fixtures/package_invalid_yaml", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-skill", "path": "tests/fixtures/package_missing_interface_field", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" } ], + "actionable_stale_skills": [], "owner_review_gaps": [ { "name": "frontend-review", @@ -672,7 +795,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -681,7 +806,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -690,7 +817,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "note-cleanup", @@ -699,7 +828,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -708,7 +839,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "geo-ranking-article-generator", @@ -717,7 +850,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "embedded-generated-skill" }, { "name": "invalid-governance-skill", @@ -725,7 +860,9 @@ "missing": [ "owner", "review_cadence" - ] + ], + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-yaml-skill", @@ -734,7 +871,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-skill", @@ -743,9 +882,12 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "test-fixture" } ], + "actionable_owner_review_gaps": [], "no_route_opportunities": [], "artifacts": { "catalog": "skill_atlas/catalog.json", diff --git a/reports/system-model.json b/reports/system-model.json index 40e537b..bd11096 100644 --- a/reports/system-model.json +++ b/reports/system-model.json @@ -3,21 +3,38 @@ "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "systems_doctrine": "Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight.", "stability": { - "score": 86, - "band": "stable-first-pass" + "score": 100, + "band": "system-ready" }, "boundary_map": { - "owned_job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "owned_job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "input_boundary": [ - "user-provided workflow notes, prompts, docs, or examples" + "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions", + "an existing skill directory that needs refactor, evaluation, packaging, or governance hardening", + "target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution", + "benchmark references, local constraints, desired maturity tier, and review standards" ], - "output_boundary": "a reusable skill output", + "output_boundary": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.", "non_goals": [ - "one-off adjacent requests that do not match the recurring job", - "private local material that was not intentionally included" + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill", + "raw private material that was not intentionally supplied as skill evidence", + "platform-specific plugin creation when the request is not about an agent skill package" + ], + "constraints": [ + "keep SKILL.md lean and route primarily through frontmatter description", + "put durable guidance in references, executable logic in scripts, and evidence in reports", + "default to the lightest reliable mode before adding governance weight", + "preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets", + "avoid raw prompt, output, transcript, or private content in telemetry" + ], + "standards": [ + "trigger boundaries must be tested with should-trigger and should-not-trigger cases", + "production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence", + "governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation", + "generated reports should be bilingual or reviewer-friendly when they are user-facing", + "each new asset must earn its place by reducing ambiguity, risk, or repeated work" ], - "constraints": [], - "standards": [], "human_judgment_boundary": [ "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", @@ -28,7 +45,7 @@ "feedback_loops": [ { "name": "Intent boundary loop", - "signal": "Intent confidence score is 30/100.", + "signal": "Intent confidence score is 100/100.", "response": "Ask only the highest-leverage clarification before adding package weight.", "evidence": "reports/intent-confidence.md and reports/intent-dialogue.md" }, @@ -38,8 +55,8 @@ "response": "Borrow one pattern at a time and keep the rest as reviewer-visible evidence.", "evidence": "reports/reference-synthesis.md", "current_patterns": [ - "Borrow a first-time operator flow that explains itself before it asks for more structure.", - "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", + "Borrow a review checkpoint wherever trust matters more than raw speed.", "Borrow the discipline of defining what the skill should not own before growing the package.", "Borrow the way it turns a messy workflow into a repeatable operating path.", "Borrow the clear execution entrypoints and command structure." @@ -131,18 +148,10 @@ "family": "Prompt-behavior mismatch", "symptom": "The role, task, and format are copied from a prompt instead of becoming stable skill behavior.", "repair": "Convert reusable role/task/format assumptions into workflow, reports, or references.", - "watch_axes": [ - "Specificity" - ] + "watch_axes": [] } ], "leverage_points": [ - { - "rank": 1, - "point": "Clarify the real job boundary", - "why": "Intent uncertainty creates downstream trigger, output, and governance errors.", - "move": "Ask one focused question and update intent context before adding assets." - }, { "rank": 2, "point": "Tune the frontmatter description", @@ -159,7 +168,7 @@ "rank": 4, "point": "Borrow one pattern, not a whole product", "why": "External references improve quality when reduced to structure, not copied as surface style.", - "move": "Start from: Borrow a first-time operator flow that explains itself before it asks for more structure." + "move": "Start from: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts." }, { "rank": 5, diff --git a/reports/system-model.md b/reports/system-model.md index 85de725..4be3a0f 100644 --- a/reports/system-model.md +++ b/reports/system-model.md @@ -2,20 +2,37 @@ Skill: `yao-meta-skill` -- Stability score: `86/100` -- Stability band: `stable-first-pass` +- Stability score: `100/100` +- Stability band: `system-ready` - Doctrine: Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight. ## System Boundary Map -- Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. -- Output boundary: a reusable skill output +- Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. +- Output boundary: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations. - Maturity assumption: `governed` - Input boundary: - - user-provided workflow notes, prompts, docs, or examples + - rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions + - an existing skill directory that needs refactor, evaluation, packaging, or governance hardening + - target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution + - benchmark references, local constraints, desired maturity tier, and review standards - Non-goals: - - one-off adjacent requests that do not match the recurring job - - private local material that was not intentionally included + - one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill + - general code review or debugging unless the user is packaging that workflow as a skill + - raw private material that was not intentionally supplied as skill evidence + - platform-specific plugin creation when the request is not about an agent skill package +- Constraints: + - keep SKILL.md lean and route primarily through frontmatter description + - put durable guidance in references, executable logic in scripts, and evidence in reports + - default to the lightest reliable mode before adding governance weight + - preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets + - avoid raw prompt, output, transcript, or private content in telemetry +- Standards: + - trigger boundaries must be tested with should-trigger and should-not-trigger cases + - production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence + - governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation + - generated reports should be bilingual or reviewer-friendly when they are user-facing + - each new asset must earn its place by reducing ambiguity, risk, or repeated work - Human judgment boundary: - Ask one focused clarification when the real job, output, or exclusion boundary is unclear. - Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints. @@ -25,7 +42,7 @@ Skill: `yao-meta-skill` ### Intent boundary loop -- Signal: Intent confidence score is 30/100. +- Signal: Intent confidence score is 100/100. - Response: Ask only the highest-leverage clarification before adding package weight. - Evidence: reports/intent-confidence.md and reports/intent-dialogue.md @@ -35,8 +52,8 @@ Skill: `yao-meta-skill` - Response: Borrow one pattern at a time and keep the rest as reviewer-visible evidence. - Evidence: reports/reference-synthesis.md - Current patterns: - - Borrow a first-time operator flow that explains itself before it asks for more structure. - - Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. + - Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. + - Borrow a review checkpoint wherever trust matters more than raw speed. - Borrow the discipline of defining what the skill should not own before growing the package. - Borrow the way it turns a messy workflow into a repeatable operating path. - Borrow the clear execution entrypoints and command structure. @@ -124,16 +141,9 @@ Skill: `yao-meta-skill` - Symptom: The role, task, and format are copied from a prompt instead of becoming stable skill behavior. - Repair: Convert reusable role/task/format assumptions into workflow, reports, or references. -- Watch Axes: - - Specificity ## Highest Leverage Moves -### 1. Clarify the real job boundary - -- Why: Intent uncertainty creates downstream trigger, output, and governance errors. -- Move: Ask one focused question and update intent context before adding assets. - ### 2. Tune the frontmatter description - Why: The description is the highest-leverage routing surface. @@ -147,7 +157,7 @@ Skill: `yao-meta-skill` ### 4. Borrow one pattern, not a whole product - Why: External references improve quality when reduced to structure, not copied as surface style. -- Move: Start from: Borrow a first-time operator flow that explains itself before it asks for more structure. +- Move: Start from: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. ### 5. Close the lifecycle loop diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json new file mode 100644 index 0000000..4f87e44 --- /dev/null +++ b/reports/upgrade_check.json @@ -0,0 +1,94 @@ +{ + "ok": true, + "schema_version": "2.0", + "generated_at": "2026-06-13", + "previous": { + "name": "yao-meta-skill", + "version": "1.0.0", + "targets": [ + "openai", + "claude", + "generic" + ] + }, + "current": { + "name": "yao-meta-skill", + "version": "1.1.0", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ] + }, + "summary": { + "declared_bump": "minor", + "recommended_bump": "minor", + "breaking_change_count": 0, + "added_target_count": 1, + "metadata_change_count": 1, + "checksum_change_count": 2, + "failure_count": 0, + "warning_count": 0 + }, + "upgrade_diff": { + "name_changed": false, + "added_targets": [ + "agent-skills-compatible" + ], + "removed_targets": [], + "compatibility_changes": [ + { + "target": "agent-skills", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "agent-skills-compatible", + "from": "missing", + "to": "pass", + "direction": "improved" + }, + { + "target": "vscode", + "from": "missing", + "to": "pass", + "direction": "improved" + } + ], + "metadata_changes": [ + { + "field": "description", + "from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.", + "to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + } + ], + "checksum_changes": [ + { + "field": "archive_sha256", + "from": "", + "to": "4d1227dc0996a97aa4987761d46081391fe15b36362e19765e1e7aa8e5d41c06" + }, + { + "field": "package_sha256", + "from": "0000000000000000000000000000000000000000000000000000000000000000", + "to": "9f1d53dbaeb1c90c80d7ffbbc1d10473e50b8032acda68332fbae551bb246abe" + } + ] + }, + "release_notes": [ + "Recommended version bump: minor.", + "Added targets: agent-skills-compatible.", + "Package or archive checksum changed; reviewers should verify package artifacts before release.", + "Package verification evidence: reports/package_verification.md." + ], + "failures": [], + "warnings": [], + "artifacts": { + "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", + "current_package": "reports/registry_audit.json", + "json": "reports/upgrade_check.json", + "markdown": "reports/upgrade_check.md" + } +} diff --git a/reports/upgrade_check.md b/reports/upgrade_check.md new file mode 100644 index 0000000..2eb27d7 --- /dev/null +++ b/reports/upgrade_check.md @@ -0,0 +1,31 @@ +# Upgrade Check + +- OK: `True` +- Previous: `yao-meta-skill 1.0.0` +- Current: `yao-meta-skill 1.1.0` +- Declared bump: `minor` +- Recommended bump: `minor` +- Breaking changes: `0` + +## Upgrade Diff + +- Added targets: `agent-skills-compatible` +- Removed targets: `none` +- Compatibility changes: `3` +- Metadata changes: `1` +- Checksum changes: `2` + +## Release Notes + +- Recommended version bump: minor. +- Added targets: agent-skills-compatible. +- Package or archive checksum changed; reviewers should verify package artifacts before release. +- Package verification evidence: reports/package_verification.md. + +## Failures + +- None + +## Warnings + +- None diff --git a/scripts/adjudicate_output_review.py b/scripts/adjudicate_output_review.py new file mode 100644 index 0000000..759bc53 --- /dev/null +++ b/scripts/adjudicate_output_review.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_BLIND_PACK = ROOT / "reports" / "output_blind_review_pack.json" +DEFAULT_ANSWER_KEY = ROOT / "reports" / "output_blind_answer_key.json" +DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json" +DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_adjudication.json" +DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_adjudication.md" + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> tuple[dict[str, Any], list[str]]: + if not path.exists(): + return {}, [f"Missing JSON file: {display_path(path)}"] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return {}, [f"Invalid JSON file {display_path(path)}: {exc}"] + if not isinstance(payload, dict): + return {}, [f"JSON root must be an object: {display_path(path)}"] + return payload, [] + + +def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]: + if not path.exists(): + return {"schema_version": "1.0", "decisions": []}, [] + return load_json(path) + + +def normalize_variant(value: Any) -> str: + normalized = str(value or "").strip().upper() + if normalized in {"VARIANT A", "A"}: + return "A" + if normalized in {"VARIANT B", "B"}: + return "B" + return normalized + + +def confidence_value(value: Any) -> tuple[float | None, str | None]: + if value is None or value == "": + return None, None + try: + parsed = float(value) + except (TypeError, ValueError): + return None, f"confidence must be numeric, got {value!r}" + if parsed < 0 or parsed > 1: + return None, f"confidence must be between 0 and 1, got {parsed}" + return round(parsed, 3), None + + +def answer_index(answer_key: dict[str, Any]) -> dict[str, dict[str, Any]]: + answers = answer_key.get("answers", []) + if not isinstance(answers, list): + return {} + return {str(item.get("case_id", "")): item for item in answers if isinstance(item, dict) and item.get("case_id")} + + +def pair_index(blind_pack: dict[str, Any]) -> dict[str, dict[str, Any]]: + pairs = blind_pack.get("pairs", []) + if not isinstance(pairs, list): + return {} + return {str(item.get("case_id", "")): item for item in pairs if isinstance(item, dict) and item.get("case_id")} + + +def decision_index(decisions_payload: dict[str, Any]) -> tuple[dict[str, dict[str, Any]], list[str]]: + decisions = decisions_payload.get("decisions", []) + failures: list[str] = [] + indexed: dict[str, dict[str, Any]] = {} + if not isinstance(decisions, list): + return {}, ["decisions must be a list"] + for index, item in enumerate(decisions, start=1): + if not isinstance(item, dict): + failures.append(f"decision #{index} must be an object") + continue + case_id = str(item.get("case_id", "")).strip() + if not case_id: + failures.append(f"decision #{index} is missing case_id") + continue + if case_id in indexed: + failures.append(f"duplicate decision for case_id: {case_id}") + continue + indexed[case_id] = item + return indexed, failures + + +def build_decision_template(blind_pack: dict[str, Any]) -> dict[str, Any]: + pairs = blind_pack.get("pairs", []) + template_decisions = [] + if isinstance(pairs, list): + for pair in pairs: + if not isinstance(pair, dict): + continue + template_decisions.append( + { + "case_id": str(pair.get("case_id", "")), + "winner_variant": "", + "confidence": None, + "reason": "", + } + ) + return { + "schema_version": "1.0", + "reviewer": "", + "reviewed_at": "", + "decision_contract": { + "winner_variant": "Use A or B after reading the blind review pack. Leave blank when pending.", + "confidence": "Optional number from 0 to 1.", + "reason": "Short reviewer rationale. Do not reveal baseline or with-skill labels before adjudication.", + }, + "decisions": template_decisions, + } + + +def adjudicate_pair( + case_id: str, + pair: dict[str, Any], + answer: dict[str, Any] | None, + decision: dict[str, Any] | None, +) -> tuple[dict[str, Any], list[str]]: + failures: list[str] = [] + expected = normalize_variant(answer.get("expected_winner_variant", "") if answer else "") + if expected not in {"A", "B"}: + failures.append(f"{case_id}: answer key is missing expected_winner_variant") + if decision is None: + return ( + { + "case_id": case_id, + "status": "pending", + "expected_winner_variant": expected, + "reviewer_winner_variant": "", + "confidence": None, + "reason": "", + "prompt": str(pair.get("prompt", "")), + }, + failures, + ) + + reviewer = normalize_variant( + decision.get("winner_variant", decision.get("reviewer_winner_variant", decision.get("winner", ""))) + ) + confidence, confidence_failure = confidence_value(decision.get("confidence")) + reason = str(decision.get("reason", "")).strip() + if not reviewer: + return ( + { + "case_id": case_id, + "status": "pending", + "expected_winner_variant": expected, + "reviewer_winner_variant": "", + "confidence": confidence, + "reason": reason, + "prompt": str(pair.get("prompt", "")), + }, + failures, + ) + if reviewer not in {"A", "B"}: + failures.append(f"{case_id}: winner_variant must be A or B") + status = "invalid" + elif confidence_failure: + failures.append(f"{case_id}: {confidence_failure}") + status = "invalid" + else: + status = "match" if reviewer == expected else "disagree" + return ( + { + "case_id": case_id, + "status": status, + "expected_winner_variant": expected, + "reviewer_winner_variant": reviewer, + "confidence": confidence, + "reason": reason, + "prompt": str(pair.get("prompt", "")), + }, + failures, + ) + + +def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, Any]: + pair_count = len(pairs) + judgment_count = sum(1 for item in pairs if item["status"] in {"match", "disagree"}) + pending_count = sum(1 for item in pairs if item["status"] == "pending") + agreement_count = sum(1 for item in pairs if item["status"] == "match") + disagreement_count = sum(1 for item in pairs if item["status"] == "disagree") + invalid_decision_count = sum(1 for item in pairs if item["status"] == "invalid") + agreement_rate = round(agreement_count / judgment_count * 100, 2) if judgment_count else None + return { + "pair_count": pair_count, + "judgment_count": judgment_count, + "pending_count": pending_count, + "agreement_count": agreement_count, + "disagreement_count": disagreement_count, + "invalid_decision_count": invalid_decision_count, + "agreement_rate": agreement_rate, + "needs_review": pending_count > 0, + "failure_count": len(failures), + } + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + "# Output Review Adjudication", + "", + "This report adjudicates reviewer choices from the blind A/B output review pack against the separate answer key.", + "", + f"- Pairs: `{summary['pair_count']}`", + f"- Judgments: `{summary['judgment_count']}`", + f"- Pending: `{summary['pending_count']}`", + f"- Agreement rate: `{summary['agreement_rate'] if summary['agreement_rate'] is not None else 'n/a'}`", + f"- Invalid decisions: `{summary['invalid_decision_count']}`", + "", + ] + if summary["judgment_count"] == 0: + lines.extend( + [ + "No reviewer decisions recorded yet.", + "", + "Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.", + "", + ] + ) + lines.extend( + [ + "## Case Adjudication", + "", + "| Case | Reviewer | Expected | Status | Confidence | Reason |", + "| --- | --- | --- | --- | ---: | --- |", + ] + ) + for item in payload["pairs"]: + confidence = "" if item.get("confidence") is None else str(item["confidence"]) + reason = str(item.get("reason", "")).replace("|", "\\|") or "" + lines.append( + f"| {item['case_id']} | {item.get('reviewer_winner_variant', '') or 'pending'} | " + f"{item.get('expected_winner_variant', '') or 'missing'} | {item['status']} | {confidence} | {reason} |" + ) + if payload.get("failures"): + lines.extend(["", "## Failures", ""]) + for failure in payload["failures"]: + lines.append(f"- {failure}") + lines.extend( + [ + "", + "## Next Fixes", + "", + "- Keep the blind review pack separate from the answer key until decisions are recorded.", + "- Treat disagreement cases as prompts for rubric tuning or output improvement.", + "- Add model-executed holdout runs after this human adjudication harness is stable.", + ] + ) + return "\n".join(lines).strip() + "\n" + + +def adjudicate_output_review( + blind_pack_path: Path, + answer_key_path: Path, + decisions_path: Path, + output_json: Path, + output_md: Path, + write_template: bool = False, +) -> dict[str, Any]: + blind_pack, failures = load_json(blind_pack_path) + answer_key, answer_failures = load_json(answer_key_path) + failures.extend(answer_failures) + decisions_payload, decision_failures = load_optional_decisions(decisions_path) + failures.extend(decision_failures) + + template_written = False + if write_template and blind_pack and not decisions_path.exists(): + decisions_path.parent.mkdir(parents=True, exist_ok=True) + decisions_path.write_text( + json.dumps(build_decision_template(blind_pack), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + template_written = True + + pairs_by_id = pair_index(blind_pack) + answers_by_id = answer_index(answer_key) + decisions_by_id, index_failures = decision_index(decisions_payload) + failures.extend(index_failures) + + for case_id in decisions_by_id: + if case_id not in pairs_by_id: + failures.append(f"decision references unknown case_id: {case_id}") + + adjudicated_pairs: list[dict[str, Any]] = [] + for case_id, pair in pairs_by_id.items(): + adjudicated, pair_failures = adjudicate_pair(case_id, pair, answers_by_id.get(case_id), decisions_by_id.get(case_id)) + adjudicated_pairs.append(adjudicated) + failures.extend(pair_failures) + + summary = build_summary(adjudicated_pairs, failures) + payload = { + "schema_version": "1.0", + "ok": not failures, + "summary": summary, + "reviewer": decisions_payload.get("reviewer", ""), + "reviewed_at": decisions_payload.get("reviewed_at", ""), + "artifacts": { + "blind_pack": display_path(blind_pack_path), + "answer_key": display_path(answer_key_path), + "decisions": display_path(decisions_path), + "json": display_path(output_json), + "markdown": display_path(output_md), + }, + "template_written": template_written, + "pairs": adjudicated_pairs, + "failures": failures, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Adjudicate blind A/B output review decisions against the answer key.") + parser.add_argument("--blind-pack", default=str(DEFAULT_BLIND_PACK)) + parser.add_argument("--answer-key", default=str(DEFAULT_ANSWER_KEY)) + parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS)) + parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON)) + parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD)) + parser.add_argument("--write-template", action="store_true") + args = parser.parse_args() + + payload = adjudicate_output_review( + Path(args.blind_pack).resolve(), + Path(args.answer_key).resolve(), + Path(args.decisions).resolve(), + Path(args.output_json).resolve(), + Path(args.output_md).resolve(), + write_template=args.write_template, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_skill_atlas.py b/scripts/build_skill_atlas.py index fec8799..4e1b3f4 100644 --- a/scripts/build_skill_atlas.py +++ b/scripts/build_skill_atlas.py @@ -52,6 +52,11 @@ CADENCE_DAYS = { "annual": 370, "per-release": 120, } +DEFAULT_SCOPE = { + "scope": "release", + "actionable": True, + "scope_reason": "default release-actionable skill", +} def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]: @@ -86,6 +91,20 @@ def load_json(path: Path) -> dict[str, Any]: return payload if isinstance(payload, dict) else {} +def load_scope_policy(workspace_root: Path) -> dict[str, Any]: + path = workspace_root / "skill_atlas" / "policy.json" + if not path.exists(): + return {"present": False, "path": safe_rel(workspace_root, path), "rules": []} + payload = load_json(path) + rules = payload.get("scope_rules", []) + return { + "present": True, + "path": safe_rel(workspace_root, path), + "schema_version": str(payload.get("schema_version", "")), + "rules": rules if isinstance(rules, list) else [], + } + + def should_skip(path: Path, root: Path) -> bool: try: rel = path.relative_to(root) @@ -124,6 +143,29 @@ def safe_rel(root: Path, path: Path) -> str: return str(path.resolve()) +def path_matches_prefix(rel_path: str, prefix: str) -> bool: + normalized_path = rel_path.strip("/") + normalized_prefix = prefix.strip("/") + if not normalized_prefix: + return False + return normalized_path == normalized_prefix or normalized_path.startswith(normalized_prefix + "/") + + +def scope_for_path(rel_path: str, policy: dict[str, Any]) -> dict[str, Any]: + for rule in policy.get("rules", []): + if not isinstance(rule, dict): + continue + prefix = str(rule.get("path_prefix", "")).strip() + if not prefix or not path_matches_prefix(rel_path, prefix): + continue + return { + "scope": str(rule.get("scope") or "supporting"), + "actionable": bool(rule.get("actionable", False)), + "scope_reason": str(rule.get("reason") or f"matched policy prefix {prefix}"), + } + return dict(DEFAULT_SCOPE) + + def display_path(path: Path) -> str: try: return str(path.resolve().relative_to(ROOT.resolve())) @@ -148,15 +190,17 @@ def resource_names(skill_dir: Path) -> list[str]: return names -def collect_skill(workspace_root: Path, skill_dir: Path) -> dict[str, Any]: +def collect_skill(workspace_root: Path, skill_dir: Path, policy: dict[str, Any]) -> dict[str, Any]: frontmatter, _ = parse_frontmatter(skill_dir / "SKILL.md") manifest = load_json(skill_dir / "manifest.json") name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name) description = str(frontmatter.get("description") or "") targets = manifest.get("target_platforms", []) + rel_path = safe_rel(workspace_root, skill_dir) + scope = scope_for_path(rel_path, policy) return { "name": name, - "path": safe_rel(workspace_root, skill_dir), + "path": rel_path, "description": description, "owner": str(manifest.get("owner", "")), "version": str(manifest.get("version", "")), @@ -167,6 +211,9 @@ def collect_skill(workspace_root: Path, skill_dir: Path) -> dict[str, Any]: "targets": [str(item) for item in targets] if isinstance(targets, list) else [], "resources": resource_names(skill_dir), "token_set": sorted(tokens(description)), + "atlas_scope": scope["scope"], + "actionable": scope["actionable"], + "scope_reason": scope["scope_reason"], } @@ -184,6 +231,9 @@ def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[ "path_b": right["path"], "score": score, "status": status, + "actionable": bool(left.get("actionable") and right.get("actionable")), + "scope_a": str(left.get("atlas_scope", "")), + "scope_b": str(right.get("atlas_scope", "")), } rows.append(row) if status == "collision": @@ -202,6 +252,19 @@ def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[ "path_b": item["paths"][1], "score": 1.0, "status": "duplicate-name", + "actionable": all( + skill.get("actionable") + for skill in skills + if skill["name"] == item["name"] and skill["path"] in set(item["paths"][:2]) + ), + "scope_a": next( + (str(skill.get("atlas_scope", "")) for skill in skills if skill["path"] == item["paths"][0]), + "", + ), + "scope_b": next( + (str(skill.get("atlas_scope", "")) for skill in skills if skill["path"] == item["paths"][1]), + "", + ), } ) return rows, collisions @@ -239,7 +302,15 @@ def stale_skills(skills: list[dict[str, Any]], today: date) -> list[dict[str, An cadence = skill.get("review_cadence") or "" allowed_days = CADENCE_DAYS.get(cadence, 120) if not updated: - stale.append({"name": skill["name"], "path": skill["path"], "reason": "missing updated_at"}) + stale.append( + { + "name": skill["name"], + "path": skill["path"], + "reason": "missing updated_at", + "actionable": bool(skill.get("actionable")), + "scope": str(skill.get("atlas_scope", "")), + } + ) continue age = (today - updated).days if age > allowed_days: @@ -250,6 +321,8 @@ def stale_skills(skills: list[dict[str, Any]], today: date) -> list[dict[str, An "reason": f"review overdue by cadence {cadence or 'unspecified'}", "age_days": age, "allowed_days": allowed_days, + "actionable": bool(skill.get("actionable")), + "scope": str(skill.get("atlas_scope", "")), } ) return stale @@ -266,7 +339,15 @@ def owner_review_gaps(skills: list[dict[str, Any]]) -> list[dict[str, Any]]: if not skill.get("maturity"): missing.append("maturity") if missing: - gaps.append({"name": skill["name"], "path": skill["path"], "missing": missing}) + gaps.append( + { + "name": skill["name"], + "path": skill["path"], + "missing": missing, + "actionable": bool(skill.get("actionable")), + "scope": str(skill.get("atlas_scope", "")), + } + ) return gaps @@ -288,7 +369,7 @@ def no_route_opportunities(workspace_root: Path) -> list[dict[str, Any]]: def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: path.parent.mkdir(parents=True, exist_ok=True) - fields = ["skill_a", "skill_b", "path_a", "path_b", "score", "status"] + fields = ["skill_a", "skill_b", "path_a", "path_b", "score", "status", "actionable", "scope_a", "scope_b"] buffer = StringIO() writer = csv.DictWriter(buffer, fieldnames=fields, lineterminator="\n") writer.writeheader() @@ -308,9 +389,10 @@ def render_html(payload: dict[str, Any]) -> str: f"{html.escape(skill.get('owner') or 'missing')}" f"{html.escape(skill.get('maturity') or 'unknown')}" f"{html.escape(skill.get('review_cadence') or 'missing')}" + f"{html.escape(skill.get('atlas_scope') or 'release')}" "" ) - blockers = payload["route_collisions"][:20] + payload["owner_review_gaps"][:20] + payload["stale_skills"][:20] + blockers = payload["actionable_route_collisions"][:20] + payload["actionable_owner_review_gaps"][:20] + payload["actionable_stale_skills"][:20] blocker_items = "".join( f"
  • {html.escape(item.get('name', item.get('skill_a', 'issue')))} {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', [])))))}
    • " for item in blockers @@ -342,19 +424,24 @@ def render_html(payload: dict[str, Any]) -> str:

    Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

    Skills{summary['skill_count']}
    -
    Route Collisions{summary['route_collision_count']}
    -
    Owner Gaps{summary['owner_gap_count']}
    -
    Stale Skills{summary['stale_count']}
    +
    Actionable{summary['actionable_skill_count']}
    +
    Route Collisions{summary['actionable_route_collision_count']}
    +
    Owner Gaps{summary['actionable_owner_gap_count']}
    +
    Stale Skills{summary['actionable_stale_count']}
    No-Route Opportunities{summary['no_route_opportunity_count']}
    -

    Top Issues

    +

    Actionable Issues

      {blocker_items or '
    • No blocking portfolio issues detected.
      • '}
    +
    +

    Full Portfolio Counts

    +

    All scanned skills remain visible: {summary['route_collision_count']} total route collisions, {summary['owner_gap_count']} total owner gaps, and {summary['stale_count']} total stale signals.

    +

    Catalog

    - + {''.join(rows)}
    NamePathOwnerMaturityReview
    NamePathOwnerMaturityReviewScope
    @@ -366,20 +453,32 @@ def render_html(payload: dict[str, Any]) -> str: def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, report_json: Path, threshold: float, today: date) -> dict[str, Any]: workspace_root = workspace_root.resolve() + scope_policy = load_scope_policy(workspace_root) skill_dirs = find_skill_dirs(workspace_root) - skills = [collect_skill(workspace_root, skill_dir) for skill_dir in skill_dirs] + skills = [collect_skill(workspace_root, skill_dir, scope_policy) for skill_dir in skill_dirs] overlap_rows, collisions = route_overlap(skills, threshold) graph = dependency_graph(skills) stale = stale_skills(skills, today) owner_gaps = owner_review_gaps(skills) opportunities = no_route_opportunities(workspace_root) + actionable_skills = [skill for skill in skills if skill.get("actionable")] + actionable_collisions = [item for item in collisions if item.get("actionable")] + actionable_stale = [item for item in stale if item.get("actionable")] + actionable_owner_gaps = [item for item in owner_gaps if item.get("actionable")] summary = { "skill_count": len(skills), + "actionable_skill_count": len(actionable_skills), "route_collision_count": len(collisions), + "actionable_route_collision_count": len(actionable_collisions), "owner_gap_count": len(owner_gaps), + "actionable_owner_gap_count": len(actionable_owner_gaps), "stale_count": len(stale), + "actionable_stale_count": len(actionable_stale), "shared_resource_count": len(graph["shared_resources"]), "no_route_opportunity_count": len(opportunities), + "non_actionable_issue_count": (len(collisions) - len(actionable_collisions)) + + (len(owner_gaps) - len(actionable_owner_gaps)) + + (len(stale) - len(actionable_stale)), } catalog = { "workspace_root": display_path(workspace_root), @@ -391,11 +490,15 @@ def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, repor "ok": True, "workspace_root": display_path(workspace_root), "summary": summary, + "scope_policy": scope_policy, "catalog": catalog, "route_collisions": collisions, + "actionable_route_collisions": actionable_collisions, "dependency_graph": graph, "stale_skills": stale, + "actionable_stale_skills": actionable_stale, "owner_review_gaps": owner_gaps, + "actionable_owner_review_gaps": actionable_owner_gaps, "no_route_opportunities": opportunities, "artifacts": { "catalog": display_path(output_dir / "catalog.json"), diff --git a/scripts/ci_test.py b/scripts/ci_test.py index d01e1cf..fda84fe 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -34,13 +34,24 @@ DEFAULT_TARGETS = [ "skill-report-metrics-check", "skill-report-charts-check", "skill-ir-check", + "compiler-check", "output-eval-check", + "output-execution-check", + "output-review-adjudication-check", "runtime-conformance-check", + "runtime-permission-check", "trust-check", "skill-atlas-check", + "registry-audit-check", + "package-verify-check", + "install-simulation-check", + "upgrade-check", "review-viewer-check", "review-studio-check", "feedback-check", + "adoption-drift-check", + "review-waivers-check", + "review-annotations-check", "baseline-compare-check", "reference-scan-check", "github-benchmark-scan-check", diff --git a/scripts/compile_skill.py b/scripts/compile_skill.py new file mode 100644 index 0000000..62c5993 --- /dev/null +++ b/scripts/compile_skill.py @@ -0,0 +1,726 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent +COMPILER_SCHEMA_VERSION = "1.0" +COMPILER_NAME = "yao-skill-ir-compiler" + + +TARGET_TRANSFORMS: dict[str, dict[str, Any]] = { + "openai": { + "adapter_mode": "metadata-adapter", + "generated_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], + "metadata_mapping": { + "display_name": "targets/openai/agents/openai.yaml::interface.display_name", + "default_prompt": "targets/openai/agents/openai.yaml::interface.default_prompt", + "activation": "targets/openai/agents/openai.yaml::compatibility.activation_mode", + "execution": "targets/openai/agents/openai.yaml::compatibility.execution_context", + "trust": "targets/openai/agents/openai.yaml::compatibility.trust_level", + "permissions": "targets/openai/agents/openai.yaml::compatibility.permission_contract", + "degradation": "targets/openai/agents/openai.yaml::compatibility.degradation_strategy", + }, + "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": ["client-native script permission prompts are represented as permission contract metadata"], + }, + "claude": { + "adapter_mode": "neutral-source-plus-adapter", + "generated_files": ["targets/claude/adapter.json", "targets/claude/README.md"], + "metadata_mapping": { + "display_name": "targets/claude/adapter.json::display_name", + "default_prompt": "targets/claude/adapter.json::default_prompt", + "activation": "targets/claude/adapter.json::activation_mode", + "execution": "targets/claude/adapter.json::execution_context", + "trust": "targets/claude/adapter.json::trust_level", + "permissions": "targets/claude/adapter.json::target_permission_contract", + "degradation": "targets/claude/adapter.json::degradation_strategy", + }, + "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": ["vendor-native metadata fields are carried as adapter JSON and README notes"], + }, + "generic": { + "adapter_mode": "agent-skills-compatible", + "generated_files": ["targets/generic/adapter.json"], + "metadata_mapping": { + "display_name": "targets/generic/adapter.json::display_name", + "default_prompt": "targets/generic/adapter.json::default_prompt", + "activation": "targets/generic/adapter.json::activation_mode", + "execution": "targets/generic/adapter.json::execution_context", + "trust": "targets/generic/adapter.json::trust_level", + "permissions": "targets/generic/adapter.json::target_permission_contract", + "degradation": "targets/generic/adapter.json::degradation_strategy", + }, + "preserved_semantics": ["trigger", "workflow-counts", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": [], + }, + "agent-skills-compatible": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": ["SKILL.md", "agents/interface.yaml"], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json", + }, + "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": [], + }, + "agent-skills": { + "adapter_mode": "neutral-agent-skills-source", + "generated_files": ["SKILL.md", "agents/interface.yaml"], + "metadata_mapping": { + "name": "SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + "manifest": "manifest.json", + }, + "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": [], + }, + "vscode": { + "adapter_mode": "agent-skills-compatible-project-or-user-scope", + "generated_files": ["SKILL.md", "agents/interface.yaml"], + "metadata_mapping": { + "name": "folder-name-and-SKILL.md::frontmatter.name", + "description": "SKILL.md::frontmatter.description", + "interface": "agents/interface.yaml", + }, + "preserved_semantics": ["trigger", "workflow", "resources", "eval-plan", "risk", "governance", "runtime", "trust", "permissions"], + "unsupported_features": ["VS Code installation scope is documented but not installed by this compiler"], + }, +} + + +def display_path(path: Path, root: Path = ROOT) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def read_frontmatter(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + text = path.read_text(encoding="utf-8", errors="replace") + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + if yaml is not None: + payload = yaml.safe_load(parts[1]) or {} + return payload if isinstance(payload, dict) else {} + data: dict[str, Any] = {} + for line in parts[1].splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data + + +def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]: + candidates = [ + skill_dir / "reports" / "skill-ir.json", + skill_dir / "skill-ir" / "examples" / f"{name}.json", + skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json", + ] + seen: set[Path] = set() + for path in candidates: + if path in seen: + continue + seen.add(path) + payload = load_json(path) + if payload: + return payload, display_path(path, skill_dir) + return {}, "frontmatter-fallback" + + +def count_list(payload: dict[str, Any], key: str) -> int: + value = payload.get(key, []) + return len(value) if isinstance(value, list) else 0 + + +def list_or_empty(payload: dict[str, Any], key: str) -> list[str]: + value = payload.get(key, []) + return [str(item) for item in value] if isinstance(value, list) else [] + + +def resource_counts(resources: dict[str, Any]) -> dict[str, int]: + return {key: count_list(resources, key) for key in ("references", "scripts", "assets", "reports")} + + +def eval_counts(eval_plan: dict[str, Any]) -> dict[str, int]: + return { + "trigger": count_list(eval_plan, "trigger"), + "output": count_list(eval_plan, "output"), + "adversarial": count_list(eval_plan, "adversarial"), + "baseline": 1 if eval_plan.get("baseline") else 0, + } + + +def scripts_with_flag(scripts: list[dict[str, Any]], flag: str) -> list[str]: + return [str(item.get("path")) for item in scripts if item.get(flag)] + + +def capability(name: str, scripts: list[str], review_reason: str) -> dict[str, Any]: + return { + "required": bool(scripts), + "script_count": len(scripts), + "scripts": scripts, + "review_reason": review_reason if scripts else "", + } + + +def permission_contract(skill_dir: Path) -> dict[str, Any]: + trust_path = skill_dir / "reports" / "security_trust_report.json" + trust = load_json(trust_path) + scripts = trust.get("scripts", []) if isinstance(trust.get("scripts"), list) else [] + network_scripts = scripts_with_flag(scripts, "uses_network") + file_write_scripts = scripts_with_flag(scripts, "uses_file_write") + subprocess_scripts = scripts_with_flag(scripts, "uses_subprocess") + interactive_scripts = scripts_with_flag(scripts, "uses_input") + capabilities = { + "network": capability("network", network_scripts, "Outbound hosts must match security/network_policy.json."), + "file_write": capability("file_write", file_write_scripts, "Scripts write local files or generated artifacts."), + "subprocess": capability("subprocess", subprocess_scripts, "Scripts spawn local commands and need operator review."), + "interactive": capability("interactive", interactive_scripts, "Scripts prompt for user input or credentials."), + } + declared = [name for name, item in capabilities.items() if item["required"]] + help_smoke = trust.get("help_smoke", {}) if isinstance(trust.get("help_smoke"), dict) else {} + network_policy = trust.get("network_policy", {}) if isinstance(trust.get("network_policy"), dict) else {} + summary = trust.get("summary", {}) if isinstance(trust.get("summary"), dict) else {} + return { + "schema_version": "1.0", + "source": display_path(trust_path, skill_dir) if trust else "missing-security-trust-report", + "source_available": bool(trust), + "declared_capabilities": declared, + "review_required": bool(declared), + "capabilities": capabilities, + "network_policy": { + "source": network_policy.get("path", "security/network_policy.json"), + "covered_scripts": network_policy.get("covered_scripts", []), + "missing_scripts": network_policy.get("missing_scripts", []), + "mismatches": network_policy.get("mismatches", []), + }, + "help_smoke": { + "enabled": bool(help_smoke.get("enabled")), + "checked_count": int(help_smoke.get("checked_count", 0) or 0), + "failed_count": int(help_smoke.get("failed_count", 0) or 0), + "failed_scripts": help_smoke.get("failed_scripts", []), + }, + "trust_summary": { + "secret_findings": int(summary.get("secret_findings", 0) or 0), + "network_script_count": int(summary.get("network_script_count", len(network_scripts)) or 0), + "file_write_script_count": int(summary.get("file_write_script_count", len(file_write_scripts)) or 0), + "subprocess_script_count": len(subprocess_scripts), + "interactive_script_count": int(summary.get("interactive_script_count", len(interactive_scripts)) or 0), + "help_smoke_failed_count": int(summary.get("help_smoke_failed_count", 0) or 0), + }, + } + + +TARGET_PERMISSION_MODELS = { + "openai": { + "model": "metadata-only", + "native_enforcement": False, + "representation": "targets/openai/agents/openai.yaml::compatibility.permission_contract plus adapter.json", + "operator_note": "OpenAI target carries permission metadata for reviewer visibility; host enforcement remains outside the package.", + }, + "claude": { + "model": "neutral-source-plus-adapter", + "native_enforcement": False, + "representation": "targets/claude/adapter.json::target_permission_contract and README notes", + "operator_note": "Claude-compatible package keeps permission intent in adapter metadata for install review.", + }, + "generic": { + "model": "agent-skills-compatible-metadata", + "native_enforcement": False, + "representation": "targets/generic/adapter.json::target_permission_contract", + "operator_note": "Generic target exposes permission metadata for downstream clients to enforce or review.", + }, +} + + +TARGET_NATIVE_MODELS = { + "openai": { + "native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source", + "activation_policy": "Use frontmatter description for catalog routing and targets/openai/agents/openai.yaml for display name, default prompt, and compatibility metadata.", + "resource_strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", + "script_strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", + "permission_enforcement": "metadata-only", + "install_scope": "plugin or skill package consumer", + "review_artifacts": ["targets/openai/agents/openai.yaml", "targets/openai/adapter.json", "reports/review-studio.html"], + "fallback_behavior": "If OpenAI-native metadata is ignored, the package remains readable as neutral Agent Skills source.", + "unsupported_native_features": [ + "client-native permission prompts", + "provider-executed scripts", + ], + }, + "claude": { + "native_surface": "Claude-compatible neutral source folder with adapter notes", + "activation_policy": "Use SKILL.md frontmatter description as the primary activation contract and adapter.json for review metadata.", + "resource_strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", + "script_strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", + "permission_enforcement": "metadata-fallback", + "install_scope": "user or project skill directory", + "review_artifacts": ["targets/claude/README.md", "targets/claude/adapter.json", "reports/review-studio.html"], + "fallback_behavior": "If Claude-specific metadata is not consumed, SKILL.md and references remain the source of truth.", + "unsupported_native_features": [ + "vendor-native permission enforcement", + "provider-specific execution transforms", + ], + }, + "generic": { + "native_surface": "Agent Skills compatible neutral package", + "activation_policy": "Use SKILL.md name and description; consumers decide automatic or manual activation.", + "resource_strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", + "script_strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "install_scope": "generic Agent Skills compatible root", + "review_artifacts": ["targets/generic/adapter.json", "reports/review-studio.html"], + "fallback_behavior": "Neutral source is the runtime fallback.", + "unsupported_native_features": [], + }, + "agent-skills-compatible": { + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "install_scope": "Agent Skills source root", + "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [], + }, + "agent-skills": { + "native_surface": "Agent Skills standard source tree", + "activation_policy": "Use SKILL.md frontmatter name and description for progressive disclosure.", + "resource_strategy": "Keep optional directories as relative resources next to SKILL.md.", + "script_strategy": "Scripts remain local optional resources and should advertise --help when executable.", + "permission_enforcement": "consumer-enforced-or-metadata-only", + "install_scope": "Agent Skills source root", + "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], + "fallback_behavior": "The source tree itself is the target artifact.", + "unsupported_native_features": [], + }, + "vscode": { + "native_surface": "VS Code/Copilot Agent Skills project or user scope", + "activation_policy": "Use folder name plus SKILL.md name/description; keep description under platform limits.", + "resource_strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", + "script_strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", + "permission_enforcement": "client-or-workspace-trust", + "install_scope": "VS Code user or project skills directory", + "review_artifacts": ["SKILL.md", "agents/interface.yaml", "reports/review-studio.html"], + "fallback_behavior": "If VS Code scope is not installed, use the neutral Agent Skills source.", + "unsupported_native_features": [ + "automatic VS Code installation", + ], + }, +} + + +def target_permission_contract(target: str, permissions: dict[str, Any]) -> dict[str, Any]: + model = TARGET_PERMISSION_MODELS.get( + target, + { + "model": "metadata-only", + "native_enforcement": False, + "representation": "adapter metadata", + "operator_note": "Permission semantics are preserved as metadata for reviewer visibility.", + }, + ) + return { + "schema_version": "1.0", + "target": target, + "permission_model": model["model"], + "native_enforcement": model["native_enforcement"], + "representation": model["representation"], + "review_required": bool(permissions.get("review_required")), + "declared_capabilities": permissions.get("declared_capabilities", []), + "capability_counts": { + name: item.get("script_count", 0) + for name, item in permissions.get("capabilities", {}).items() + }, + "evidence": permissions.get("source", ""), + "operator_note": model["operator_note"], + } + + +def target_native_contract( + target: str, + profile: dict[str, Any], + contract: dict[str, Any], + target_permissions: dict[str, Any], +) -> dict[str, Any]: + model = TARGET_NATIVE_MODELS.get( + target, + { + "native_surface": "adapter metadata", + "activation_policy": "Carry activation semantics as metadata for the target consumer.", + "resource_strategy": "Preserve neutral package resources.", + "script_strategy": "Expose script metadata for reviewer visibility.", + "permission_enforcement": "metadata-only", + "install_scope": "target consumer", + "review_artifacts": ["adapter.json", "reports/review-studio.html"], + "fallback_behavior": "Use neutral source package.", + "unsupported_native_features": [], + }, + ) + return { + "schema_version": "1.0", + "target": target, + "native_surface": model["native_surface"], + "activation": { + "policy": model["activation_policy"], + "trigger_description": contract.get("trigger", {}).get("description", ""), + "manual_activation_supported": True, + "automatic_activation_note": "Depends on the target client route/catalog implementation.", + }, + "resources": { + "strategy": model["resource_strategy"], + "counts": contract.get("resources", {}).get("counts", {}), + "generated_files": profile.get("generated_files", []), + }, + "scripts": { + "strategy": model["script_strategy"], + "script_count": contract.get("resources", {}).get("counts", {}).get("scripts", 0), + "help_smoke_failed_count": contract.get("permissions", {}).get("help_smoke", {}).get("failed_count", 0), + }, + "permissions": { + "enforcement": model["permission_enforcement"], + "native_enforcement": bool(target_permissions.get("native_enforcement")), + "declared_capabilities": target_permissions.get("declared_capabilities", []), + "review_required": bool(target_permissions.get("review_required")), + }, + "review": { + "artifacts": model["review_artifacts"], + "fallback_behavior": model["fallback_behavior"], + "unsupported_native_features": [ + *model.get("unsupported_native_features", []), + *profile.get("unsupported_features", []), + ], + }, + "install_scope": model["install_scope"], + } + + +def load_sources(skill_dir: Path) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + frontmatter = read_frontmatter(skill_dir / "SKILL.md") + manifest = load_json(skill_dir / "manifest.json") + interface_doc = load_yaml(skill_dir / "agents" / "interface.yaml") + name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name) + ir, ir_source = find_skill_ir(skill_dir, name) + return { + "skill_dir": skill_dir, + "frontmatter": frontmatter, + "manifest": manifest, + "interface_doc": interface_doc, + "interface": interface_doc.get("interface", {}), + "compatibility": interface_doc.get("compatibility", {}), + "ir": ir, + "ir_source": ir_source, + } + + +def declared_targets(sources: dict[str, Any]) -> list[str]: + ir = sources["ir"] + compatibility = sources["compatibility"] + manifest = sources["manifest"] + candidates = ( + ir.get("targets") + if isinstance(ir.get("targets"), list) + else manifest.get("target_platforms") + or compatibility.get("adapter_targets") + or [] + ) + targets = [] + for item in candidates: + value = str(item).strip() + if value and value not in targets: + targets.append(value) + return targets or ["generic"] + + +def semantic_source(sources: dict[str, Any]) -> dict[str, Any]: + skill_dir = sources["skill_dir"] + frontmatter = sources["frontmatter"] + manifest = sources["manifest"] + interface = sources["interface"] + compatibility = sources["compatibility"] + ir = sources["ir"] + trigger = ir.get("trigger_surface", {}) if isinstance(ir.get("trigger_surface"), dict) else {} + workflow = ir.get("workflow", {}) if isinstance(ir.get("workflow"), dict) else {} + resources = ir.get("resources", {}) if isinstance(ir.get("resources"), dict) else {} + eval_plan = ir.get("eval_plan", {}) if isinstance(ir.get("eval_plan"), dict) else {} + description = str(trigger.get("description") or frontmatter.get("description") or "") + name = str(ir.get("name") or frontmatter.get("name") or manifest.get("name") or skill_dir.name) + permissions = permission_contract(skill_dir) + return { + "name": name, + "title": str(ir.get("title") or interface.get("display_name") or name), + "version": str(manifest.get("version") or frontmatter.get("version") or "1.0.0"), + "description": description, + "job_to_be_done": str(ir.get("job_to_be_done") or description), + "trigger": { + "description": description, + "should_trigger": list_or_empty(trigger, "should_trigger"), + "should_not_trigger": list_or_empty(trigger, "should_not_trigger"), + "edge_cases": list_or_empty(trigger, "edge_cases"), + }, + "workflow": { + "steps": list_or_empty(workflow, "steps"), + "decision_points": list_or_empty(workflow, "decision_points"), + "failure_modes": list_or_empty(workflow, "failure_modes"), + }, + "resources": { + "references": list_or_empty(resources, "references"), + "scripts": list_or_empty(resources, "scripts"), + "assets": list_or_empty(resources, "assets"), + "reports": list_or_empty(resources, "reports"), + "counts": resource_counts(resources), + }, + "eval_plan": { + "trigger": list_or_empty(eval_plan, "trigger"), + "output": list_or_empty(eval_plan, "output"), + "adversarial": list_or_empty(eval_plan, "adversarial"), + "baseline": str(eval_plan.get("baseline") or ""), + "counts": eval_counts(eval_plan), + }, + "risk": ir.get("risk", {}) if isinstance(ir.get("risk"), dict) else {}, + "governance": ir.get("governance", {}) if isinstance(ir.get("governance"), dict) else {}, + "runtime": { + "activation": compatibility.get("activation", {}) if isinstance(compatibility.get("activation"), dict) else {}, + "execution": compatibility.get("execution", {}) if isinstance(compatibility.get("execution"), dict) else {}, + "trust": compatibility.get("trust", {}) if isinstance(compatibility.get("trust"), dict) else {}, + "adapter_targets": compatibility.get("adapter_targets", []), + "canonical_format": compatibility.get("canonical_format", "agent-skills"), + }, + "permissions": permissions, + "source_files_count": count_list(ir, "source_files") if ir else 0, + } + + +def compile_target_contract(skill_dir: Path, target: str) -> dict[str, Any]: + sources = load_sources(skill_dir) + compatibility = sources["compatibility"] + degradation = compatibility.get("degradation", {}) if isinstance(compatibility.get("degradation"), dict) else {} + declared = declared_targets(sources) + failures: list[str] = [] + warnings: list[str] = [] + profile = TARGET_TRANSFORMS.get(target) + if profile is None: + failures.append(f"Unsupported compiler target: {target}") + profile = { + "adapter_mode": "unsupported", + "generated_files": [], + "metadata_mapping": {}, + "preserved_semantics": [], + "unsupported_features": [], + } + if target not in declared and not (target == "agent-skills" and "agent-skills-compatible" in declared): + warnings.append(f"Target is not declared in Skill IR or interface metadata: {target}") + if not sources["ir"]: + warnings.append("Skill IR is missing; compiler used frontmatter fallback") + + contract = semantic_source(sources) + permissions = contract["permissions"] + target_permissions = target_permission_contract(target, permissions) + target_native = target_native_contract(target, profile, contract, target_permissions) + contract["target"] = target + contract["target_permission_contract"] = target_permissions + contract["target_native_contract"] = target_native + contract["degradation_strategy"] = degradation.get(target, degradation.get("generic", "neutral-source")) + contract["target_runtime"] = { + "adapter_mode": profile["adapter_mode"], + "generated_files": profile["generated_files"], + "metadata_mapping": profile["metadata_mapping"], + "preserved_semantics": profile["preserved_semantics"], + "native_surface": target_native["native_surface"], + } + + return { + "schema_version": COMPILER_SCHEMA_VERSION, + "target": target, + "status": "block" if failures else ("warn" if warnings else "pass"), + "compiler": { + "name": COMPILER_NAME, + "schema_version": COMPILER_SCHEMA_VERSION, + "source": "skill-ir" if sources["ir"] else "frontmatter-fallback", + "ir_source": sources["ir_source"], + "ir_schema_version": str(sources["ir"].get("schema_version") or "none"), + }, + "source": { + "skill_dir": display_path(sources["skill_dir"]), + "canonical_metadata": "agents/interface.yaml" if (sources["skill_dir"] / "agents" / "interface.yaml").exists() else "missing", + "declared_targets": declared, + }, + "compiled_contract": contract, + "permission_contract": permissions, + "target_permission_contract": target_permissions, + "target_native_contract": target_native, + "target_transform": { + "target": target, + "adapter_mode": profile["adapter_mode"], + "generated_files": profile["generated_files"], + "metadata_mapping": profile["metadata_mapping"], + "preserved_semantics": profile["preserved_semantics"], + "degradation_strategy": contract["degradation_strategy"], + "permission_representation": target_permissions["representation"], + "native_surface": target_native["native_surface"], + "activation_policy": target_native["activation"]["policy"], + "resource_strategy": target_native["resources"]["strategy"], + "script_strategy": target_native["scripts"]["strategy"], + "permission_enforcement": target_native["permissions"]["enforcement"], + }, + "unsupported_features": list(profile.get("unsupported_features", [])), + "warnings": warnings, + "failures": failures, + } + + +def compile_targets(skill_dir: Path, targets: list[str] | None = None, generated_at: str | None = None) -> dict[str, Any]: + sources = load_sources(skill_dir) + selected = targets or declared_targets(sources) + compiled = [compile_target_contract(skill_dir, target) for target in selected] + failures = [failure for item in compiled for failure in item["failures"]] + warnings = [warning for item in compiled for warning in item["warnings"]] + return { + "schema_version": COMPILER_SCHEMA_VERSION, + "ok": not failures, + "generated_at": generated_at or str(date.today()), + "skill_dir": display_path(Path(skill_dir).resolve()), + "summary": { + "target_count": len(compiled), + "pass_count": sum(1 for item in compiled if item["status"] == "pass"), + "warn_count": sum(1 for item in compiled if item["status"] == "warn"), + "block_count": sum(1 for item in compiled if item["status"] == "block"), + "failure_count": len(failures), + "warning_count": len(warnings), + }, + "targets": compiled, + "failures": failures, + "warnings": warnings, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Compiled Targets", + "", + f"- OK: `{report['ok']}`", + f"- Targets: `{summary['target_count']}`", + f"- Pass: `{summary['pass_count']}`", + f"- Warn: `{summary['warn_count']}`", + f"- Block: `{summary['block_count']}`", + "", + "## Target Transforms", + "", + "| Target | Status | Native Surface | Adapter Mode | Permissions | Degradation | Generated Files |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + for item in report["targets"]: + transform = item["target_transform"] + files = ", ".join(transform.get("generated_files", [])) or "none" + permissions = ", ".join(item.get("target_permission_contract", {}).get("declared_capabilities", [])) or "none" + lines.append( + f"| `{item['target']}` | `{item['status']}` | {transform.get('native_surface', '')} | `{transform.get('adapter_mode', '')}` | `{permissions}` | `{transform.get('degradation_strategy', '')}` | {files} |" + ) + lines.extend(["", "## Native Behavior Contracts", ""]) + for item in report["targets"]: + native = item.get("target_native_contract", {}) + permissions = native.get("permissions", {}) + review = native.get("review", {}) + lines.extend( + [ + f"### {item['target']}", + "", + f"- Native surface: {native.get('native_surface', '')}", + f"- Activation: {native.get('activation', {}).get('policy', '')}", + f"- Resources: {native.get('resources', {}).get('strategy', '')}", + f"- Scripts: {native.get('scripts', {}).get('strategy', '')}", + f"- Permission enforcement: `{permissions.get('enforcement', '')}`; native enforcement `{permissions.get('native_enforcement')}`", + f"- Review artifacts: {', '.join(review.get('artifacts', [])) or 'none'}", + "", + ] + ) + lines.extend(["", "## Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) + lines.extend(["", "## Warnings", ""]) + lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"]) + return "\n".join(lines).strip() + "\n" + + +def render_compile_report( + skill_dir: Path, + targets: list[str] | None = None, + output_json: Path | None = None, + output_md: Path | None = None, + generated_at: str | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports = skill_dir / "reports" + reports.mkdir(parents=True, exist_ok=True) + output_json = output_json or reports / "compiled_targets.json" + output_md = output_md or reports / "compiled_targets.md" + report = compile_targets(skill_dir, targets=targets, generated_at=generated_at) + report["artifacts"] = { + "json": display_path(output_json), + "markdown": display_path(output_md), + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compile Skill IR into target-specific semantic contracts.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--target", action="append", default=[]) + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--generated-at") + args = parser.parse_args() + + payload = render_compile_report( + Path(args.skill_dir), + targets=args.target or None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + output_md=Path(args.output_md).resolve() if args.output_md else None, + generated_at=args.generated_at, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/cross_packager.py b/scripts/cross_packager.py index ae81216..6799afe 100644 --- a/scripts/cross_packager.py +++ b/scripts/cross_packager.py @@ -6,6 +6,8 @@ import zipfile from pathlib import Path import yaml +from compile_skill import compile_target_contract + def display_path(path: Path, root: Path) -> str: try: @@ -199,10 +201,13 @@ def build_manifest(skill_dir: Path, platform: str) -> dict: ir=ir, ir_source=ir_source, ) + compiled = compile_target_contract(skill_dir, platform) + if compiled.get("failures"): + raise ValueError(f"Compiler failed for {platform}: {'; '.join(compiled['failures'])}") return { "name": semantic["name"], "description": semantic["description"], - "version": frontmatter.get("version", "1.0.0"), + "version": manifest.get("version") or frontmatter.get("version", "1.0.0"), "platform": platform, "skill_root": skill_dir.name, "job_to_be_done": semantic["job_to_be_done"], @@ -210,6 +215,14 @@ def build_manifest(skill_dir: Path, platform: str) -> dict: "ir_schema_version": semantic["ir_schema_version"], "semantic_contract": semantic["semantic_contract"], "semantic_parity": semantic["semantic_parity"], + "compiler": compiled["compiler"], + "compiled_contract": compiled["compiled_contract"], + "permission_contract": compiled["permission_contract"], + "target_permission_contract": compiled["target_permission_contract"], + "target_native_contract": compiled["target_native_contract"], + "target_transform": compiled["target_transform"], + "unsupported_features": compiled["unsupported_features"], + "compiler_warnings": compiled["warnings"], "display_name": interface.get("display_name", skill_dir.name), "short_description": interface.get("short_description", ""), "default_prompt": interface.get("default_prompt", ""), @@ -260,6 +273,9 @@ PLATFORM_CONTRACTS = { "remote_inline_execution", "degradation_strategy", "portability_profile", + "permission_contract", + "target_permission_contract", + "target_native_contract", ], "required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], "field_mapping": { @@ -292,6 +308,9 @@ PLATFORM_CONTRACTS = { "remote_inline_execution", "degradation_strategy", "portability_profile", + "permission_contract", + "target_permission_contract", + "target_native_contract", ], "required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], "field_mapping": { @@ -324,6 +343,9 @@ PLATFORM_CONTRACTS = { "remote_inline_execution", "degradation_strategy", "portability_profile", + "permission_contract", + "target_permission_contract", + "target_native_contract", ], "required_files": ["targets/generic/adapter.json"], "field_mapping": { @@ -336,6 +358,19 @@ PLATFORM_CONTRACTS = { }, } +EXCLUDED_ARCHIVE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"} + + +def should_skip_archive_path(rel_path: Path) -> bool: + parts = rel_path.parts + if any(part in EXCLUDED_ARCHIVE_PARTS for part in parts): + return True + if parts == ("reports", "telemetry_events.jsonl"): + return True + if parts and parts[0] == "tests" and any(part.startswith("tmp") for part in parts[1:]): + return True + return False + def is_relative_to(path: Path, parent: Path) -> bool: try: @@ -397,6 +432,19 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path: "shell": payload["shell"], "trust_level": payload["trust_level"], "remote_inline_execution": payload["remote_inline_execution"], + "permission_contract": { + "review_required": payload["target_permission_contract"]["review_required"], + "declared_capabilities": payload["target_permission_contract"]["declared_capabilities"], + "native_enforcement": payload["target_permission_contract"]["native_enforcement"], + "representation": payload["target_permission_contract"]["representation"], + }, + "native_contract": { + "native_surface": payload["target_native_contract"]["native_surface"], + "activation_policy": payload["target_native_contract"]["activation"]["policy"], + "resource_strategy": payload["target_native_contract"]["resources"]["strategy"], + "permission_enforcement": payload["target_native_contract"]["permissions"]["enforcement"], + "review_artifacts": payload["target_native_contract"]["review"]["artifacts"], + }, "degradation_strategy": payload["degradation_strategy"], }, }, @@ -404,8 +452,14 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path: payload["install_hint"] = f"Use the packaged skill and include targets/openai/agents/openai.yaml when the client expects OpenAI-style interface metadata." elif platform == "claude": notes = target_dir / "README.md" + native = payload["target_native_contract"] notes.write_text( - f"# Claude-Compatible Package\n\nUse `{skill_dir.name}` with its neutral source files. This target does not require vendor metadata by default.\n", + f"# Claude-Compatible Package\n\nUse `{skill_dir.name}` with its neutral source files. This target does not require vendor metadata by default.\n\n" + f"Native surface: {native['native_surface']}.\n\n" + f"Activation: {native['activation']['policy']}\n\n" + f"Resources: {native['resources']['strategy']}\n\n" + f"Scripts: {native['scripts']['strategy']}\n\n" + f"Permission metadata is preserved in `adapter.json` under `target_permission_contract` and `target_native_contract` for reviewer visibility.\n", encoding="utf-8", ) payload["install_hint"] = f"Use the packaged skill directly; this target relies on SKILL.md and optional neutral metadata." @@ -430,6 +484,9 @@ def make_zip(skill_dir: Path, out_dir: Path) -> Path: continue if is_relative_to(resolved, out_root): continue + rel_path = path.relative_to(skill_dir) + if should_skip_archive_path(rel_path): + continue zf.write(path, arcname=str(path.relative_to(skill_dir.parent))) return zip_path diff --git a/scripts/export_skill_ir.py b/scripts/export_skill_ir.py index 2f7870e..08386a6 100644 --- a/scripts/export_skill_ir.py +++ b/scripts/export_skill_ir.py @@ -36,10 +36,21 @@ KEY_REPORTS = [ "reports/skill-overview.html", "reports/output_quality_scorecard.json", "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", "reports/conformance_matrix.json", "reports/conformance_matrix.md", "reports/security_trust_report.json", "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", diff --git a/scripts/init_skill.py b/scripts/init_skill.py index 0f59add..be9b33e 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -5,11 +5,15 @@ import re from datetime import date from pathlib import Path +from compile_skill import render_compile_report from export_skill_ir import build_skill_ir, validate_ir from github_benchmark_scan import run_github_benchmark_scan from render_intent_confidence import render_intent_confidence from render_intent_dialogue import render_intent_dialogue from render_iteration_directions import render_iteration_directions +from render_adoption_drift_report import render_report as render_adoption_drift_report +from render_review_annotations import render_report as render_review_annotations +from render_review_waivers import render_report as render_review_waivers from render_artifact_design_profile import render_artifact_design_profile from render_output_risk_profile import render_output_risk_profile from render_prompt_quality_profile import render_prompt_quality_profile @@ -71,13 +75,17 @@ README_TEMPLATE = """# {title} 6. Follow the workflow steps in `SKILL.md`. 7. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. It defaults to Simplified Chinese and includes an English switch in the top right. 8. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view. -9. Open `reports/review-viewer.html` for a compact visual review of the package. -10. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. -11. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. -12. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. -13. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. -14. Review `reports/iteration-directions.md` for the three most valuable next moves. -15. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. +9. Record source-line reviewer comments in `reports/review_annotations.md` when review needs follow-up. +10. Open `reports/review-viewer.html` for a compact visual review of the package. +11. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. +12. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. +13. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. +14. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. +15. Review `reports/compiled_targets.md` to see how Skill IR compiles into OpenAI, Claude, generic, and Agent Skills compatible target contracts. +16. Review `reports/iteration-directions.md` for the three most valuable next moves. +17. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. +18. Review `reports/adoption_drift_report.md` to see local-first metadata-only adoption and drift signals. +19. Review `reports/review_waivers.md` to see human reviewer risk approvals and expiry dates. ## Honest Boundaries @@ -100,10 +108,14 @@ README_TEMPLATE = """# {title} - `reports/prompt-quality-profile.md`: prompt-facing need model, RTF mapping, complexity, and quality matrix - `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points - `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance +- `reports/compiled_targets.md`: target compiler report showing generated contracts, adapter modes, preserved semantics, warnings, and unsupported features - `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap - `reports/review-studio.html`: Review Studio 2.0 gate page for intent, trigger, output eval, context, runtime conformance, trust, atlas, and release readiness - `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps - `reports/iteration-directions.md`: the top three next iteration directions +- `reports/adoption_drift_report.md`: local-first metadata-only telemetry summary for adoption, missed triggers, bad outputs, script errors, and review drift +- `reports/review_waivers.md`: human reviewer risk approval ledger for warning acceptance and expiry +- `reports/review_annotations.md`: source-line reviewer comments linked to Review Studio gates """ @@ -223,8 +235,10 @@ def build_report_view(artifacts: dict) -> dict: "review_studio": review_studio, "message": ( f"Skill 已创建完成。建议先打开总结报告：{html_report}。" - "它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线；" + "它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线；" f"然后打开 Review Studio 2.0：{review_studio}，检查意图、触发、输出评测、运行一致性、信任和发布闸门。" + "后续 reviewer 的文件级或行级意见可以记录到 reports/review_annotations.md。" + "如需审查平台适配细节，请打开 reports/compiled_targets.md。" "报告默认使用中文简体，右上角可以切换英文版。" ), "next_action": "Open reports/skill-overview.html before editing more files.", @@ -336,7 +350,11 @@ def initialize_skill( prompt_quality_profile = render_prompt_quality_profile(root) system_model = render_system_model(root) skill_ir = render_skill_ir(root) + compiled_targets = render_compile_report(root) iteration_directions = render_iteration_directions(root) + adoption_drift = render_adoption_drift_report(root) + review_waivers = render_review_waivers(root) + review_annotations = render_review_annotations(root) overview = render_skill_overview(root) review_viewer = render_review_viewer(root) review_studio = render_review_studio(root) @@ -363,8 +381,16 @@ def initialize_skill( "system_model_md": system_model["artifacts"]["markdown"], "system_model_json": system_model["artifacts"]["json"], "skill_ir_json": skill_ir["artifacts"]["json"], + "compiled_targets_md": compiled_targets["artifacts"]["markdown"], + "compiled_targets_json": compiled_targets["artifacts"]["json"], "iteration_directions_md": iteration_directions["artifacts"]["markdown"], "iteration_directions_json": iteration_directions["artifacts"]["json"], + "adoption_drift_md": adoption_drift["artifacts"]["markdown"], + "adoption_drift_json": adoption_drift["artifacts"]["json"], + "review_waivers_md": review_waivers["artifacts"]["markdown"], + "review_waivers_json": review_waivers["artifacts"]["json"], + "review_annotations_md": review_annotations["artifacts"]["markdown"], + "review_annotations_json": review_annotations["artifacts"]["json"], "review_viewer_html": review_viewer["artifacts"]["html"], "review_viewer_json": review_viewer["artifacts"]["json"], "review_studio_html": absolute_skill_artifact(root, review_studio["artifacts"]["html"]), @@ -384,6 +410,7 @@ def initialize_skill( "reference_synthesis": reference_synthesis["summary"], "system_model": system_model["summary"], "skill_ir": skill_ir["summary"], + "compiled_targets": compiled_targets["summary"], "artifacts": artifacts, "report_view": report_view, } diff --git a/scripts/local_output_eval_runner.py b/scripts/local_output_eval_runner.py new file mode 100644 index 0000000..b87b0b2 --- /dev/null +++ b/scripts/local_output_eval_runner.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +import argparse +import json +import sys +from typing import Any + + +def estimate_tokens(text: str) -> int: + if not text: + return 0 + return max(1, round(len(text) / 4)) + + +def build_response(request: dict[str, Any], execution_kind: str, provider: str, model: str) -> dict[str, Any]: + output = str(request.get("fixture_output", "")) + prompt = str(request.get("prompt", "")) + input_tokens = estimate_tokens(prompt) + output_tokens = estimate_tokens(output) + response: dict[str, Any] = { + "output": output, + "execution_kind": execution_kind, + "provider": provider, + "usage": { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + "total_tokens": input_tokens + output_tokens, + "estimated": True, + }, + } + if model: + response["model"] = model + return response + + +def main() -> None: + parser = argparse.ArgumentParser( + description=( + "Local deterministic output-eval runner. It executes the runner contract " + "without claiming provider-backed model generation." + ) + ) + parser.add_argument("--execution-kind", choices=["command", "model"], default="command") + parser.add_argument("--provider", default="local-output-eval-runner") + parser.add_argument("--model", default="") + args = parser.parse_args() + + raw = sys.stdin.read() + if not raw.strip(): + print("runner requires a JSON request on stdin", file=sys.stderr) + raise SystemExit(2) + try: + request = json.loads(raw) + except json.JSONDecodeError as exc: + print(f"invalid JSON request: {exc}", file=sys.stderr) + raise SystemExit(2) + if not isinstance(request, dict): + print("runner request must be a JSON object", file=sys.stderr) + raise SystemExit(2) + + response = build_response(request, args.execution_kind, args.provider, args.model) + print(json.dumps(response, ensure_ascii=False)) + + +if __name__ == "__main__": + main() diff --git a/scripts/probe_runtime_permissions.py b/scripts/probe_runtime_permissions.py new file mode 100644 index 0000000..6ea2504 --- /dev/null +++ b/scripts/probe_runtime_permissions.py @@ -0,0 +1,289 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +DEFAULT_TARGETS = ["openai", "claude", "generic"] + + +def display_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + try: + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + except yaml.YAMLError: + return {} + return payload if isinstance(payload, dict) else {} + + +def expected_capabilities(skill_dir: Path) -> list[str]: + trust = load_json(skill_dir / "reports" / "security_trust_report.json") + governance = trust.get("permission_governance", {}) if isinstance(trust.get("permission_governance"), dict) else {} + required = governance.get("required_capabilities", []) + if isinstance(required, list) and required: + return sorted(str(item) for item in required) + summary = trust.get("summary", {}) if isinstance(trust.get("summary"), dict) else {} + candidates = [] + if int(summary.get("network_script_count", 0) or 0): + candidates.append("network") + if int(summary.get("file_write_script_count", 0) or 0): + candidates.append("file_write") + if any(item.get("uses_subprocess") for item in trust.get("scripts", []) if isinstance(item, dict)): + candidates.append("subprocess") + if int(summary.get("interactive_script_count", 0) or 0): + candidates.append("interactive") + return sorted(candidates) + + +def add_check(checks: list[dict[str, Any]], failures: list[str], key: str, condition: bool, detail: str) -> None: + checks.append({"key": key, "passed": condition, "detail": detail}) + if not condition: + failures.append(detail) + + +def sorted_strings(value: Any) -> list[str]: + return sorted(str(item) for item in value) if isinstance(value, list) else [] + + +def probe_openai_yaml(package_dir: Path, expected: list[str]) -> tuple[list[dict[str, Any]], list[str]]: + checks: list[dict[str, Any]] = [] + failures: list[str] = [] + path = package_dir / "targets" / "openai" / "agents" / "openai.yaml" + payload = load_yaml(path) + permission_contract = payload.get("compatibility", {}).get("permission_contract", {}) if payload else {} + add_check(checks, failures, "openai-yaml-present", bool(payload), "OpenAI permission metadata YAML is readable") + add_check( + checks, + failures, + "openai-yaml-permissions", + sorted_strings(permission_contract.get("declared_capabilities")) == expected, + "OpenAI YAML permission contract mirrors expected capabilities", + ) + add_check( + checks, + failures, + "openai-yaml-native-flag", + isinstance(permission_contract.get("native_enforcement"), bool), + "OpenAI YAML declares native_enforcement as a boolean", + ) + return checks, failures + + +def probe_target(skill_dir: Path, package_dir: Path, target: str, expected: list[str]) -> dict[str, Any]: + adapter_path = package_dir / "targets" / target / "adapter.json" + checks: list[dict[str, Any]] = [] + failures: list[str] = [] + adapter = load_json(adapter_path) + add_check(checks, failures, "adapter-present", bool(adapter), f"{target} adapter.json is readable") + + permission_contract = adapter.get("permission_contract", {}) if adapter else {} + target_contract = adapter.get("target_permission_contract", {}) if adapter else {} + add_check(checks, failures, "permission-contract-present", bool(permission_contract), f"{target} adapter includes permission_contract") + add_check(checks, failures, "target-contract-present", bool(target_contract), f"{target} adapter includes target_permission_contract") + add_check( + checks, + failures, + "source-available", + permission_contract.get("source_available") is True, + f"{target} permission_contract links to an available trust report", + ) + add_check( + checks, + failures, + "declared-capabilities-match", + sorted_strings(target_contract.get("declared_capabilities")) == expected, + f"{target} target_permission_contract mirrors expected capabilities", + ) + add_check( + checks, + failures, + "capability-counts-present", + isinstance(target_contract.get("capability_counts"), dict), + f"{target} target_permission_contract includes capability_counts", + ) + add_check( + checks, + failures, + "native-enforcement-boolean", + isinstance(target_contract.get("native_enforcement"), bool), + f"{target} target_permission_contract declares native_enforcement as a boolean", + ) + add_check( + checks, + failures, + "representation-present", + bool(str(target_contract.get("representation", "")).strip()), + f"{target} target_permission_contract declares where permission metadata is represented", + ) + add_check( + checks, + failures, + "operator-note-present", + bool(str(target_contract.get("operator_note", "")).strip()), + f"{target} target_permission_contract includes an operator_note", + ) + add_check( + checks, + failures, + "review-required-matches", + bool(target_contract.get("review_required")) == bool(expected), + f"{target} review_required matches whether capabilities are required", + ) + + yaml_checks: list[dict[str, Any]] = [] + yaml_failures: list[str] = [] + if target == "openai": + yaml_checks, yaml_failures = probe_openai_yaml(package_dir, expected) + checks.extend(yaml_checks) + failures.extend(yaml_failures) + + native = target_contract.get("native_enforcement") + metadata_fallback = native is False and bool(target_contract.get("representation")) and bool(target_contract.get("operator_note")) + assurance = "native-enforced" if native is True else ("metadata-fallback-explicit" if metadata_fallback else "missing") + residual_risks = [] + if native is False: + residual_risks.append("Client-native permission enforcement is not provided by this target; installer or operator must honor metadata.") + return { + "target": target, + "status": "pass" if not failures else "fail", + "adapter": display_path(adapter_path, skill_dir), + "permission_model": str(target_contract.get("permission_model", "")), + "native_enforcement": bool(native) if isinstance(native, bool) else None, + "metadata_fallback_explicit": metadata_fallback, + "assurance": assurance, + "declared_capabilities": sorted_strings(target_contract.get("declared_capabilities")), + "checks": checks, + "failures": failures, + "residual_risks": residual_risks, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Runtime Permission Probes", + "", + "Runtime permission probes verify that generated target adapters expose high-permission capabilities and make native-enforcement limits explicit.", + "", + "## Summary", + "", + f"- OK: `{report['ok']}`", + f"- Targets probed: `{summary['target_count']}`", + f"- Passed: `{summary['pass_count']}`", + f"- Failed: `{summary['fail_count']}`", + f"- Native enforcement targets: `{summary['native_enforcement_count']}`", + f"- Explicit metadata fallbacks: `{summary['metadata_fallback_count']}`", + f"- Required capabilities: `{', '.join(report['expected_capabilities']) or 'none'}`", + "", + "| Target | Status | Assurance | Native Enforcement | Metadata Fallback | Residual Risk |", + "| --- | --- | --- | --- | --- | --- |", + ] + for target in report["targets"]: + residual = "
    ".join(target["residual_risks"]) if target["residual_risks"] else "None" + lines.append( + f"| `{target['target']}` | `{target['status']}` | `{target['assurance']}` | " + f"`{target['native_enforcement']}` | `{target['metadata_fallback_explicit']}` | {residual} |" + ) + lines.extend(["", "## Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) + lines.extend( + [ + "", + "## Reviewer Note", + "", + "A passing probe means the target contract is explicit and auditable. It does not claim that a host client enforces permissions natively.", + ] + ) + return "\n".join(lines).strip() + "\n" + + +def probe_runtime_permissions( + skill_dir: Path, + package_dir: Path, + targets: list[str], + output_json: Path, + output_md: Path, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + package_dir = package_dir.resolve() + expected = expected_capabilities(skill_dir) + target_results = [probe_target(skill_dir, package_dir, target, expected) for target in targets] + failures = [failure for target in target_results for failure in target["failures"]] + summary = { + "target_count": len(target_results), + "pass_count": sum(1 for item in target_results if item["status"] == "pass"), + "fail_count": sum(1 for item in target_results if item["status"] == "fail"), + "native_enforcement_count": sum(1 for item in target_results if item["native_enforcement"] is True), + "metadata_fallback_count": sum(1 for item in target_results if item["metadata_fallback_explicit"]), + "residual_risk_count": sum(len(item["residual_risks"]) for item in target_results), + "required_capability_count": len(expected), + "failure_count": len(failures), + } + report = { + "schema_version": "1.0", + "ok": not failures, + "skill_dir": display_path(skill_dir, skill_dir), + "package_dir": display_path(package_dir, skill_dir), + "expected_capabilities": expected, + "summary": summary, + "targets": target_results, + "failures": failures, + "artifacts": { + "json": display_path(output_json, skill_dir), + "markdown": display_path(output_md, skill_dir), + }, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Probe generated target adapters for runtime permission enforcement metadata.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--package-dir", default="dist") + parser.add_argument("--target", action="append", choices=DEFAULT_TARGETS) + parser.add_argument("--output-json") + parser.add_argument("--output-md") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = probe_runtime_permissions( + skill_dir, + Path(args.package_dir).resolve(), + args.target or DEFAULT_TARGETS, + Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "runtime_permission_probes.json", + Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "runtime_permission_probes.md", + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/registry_audit.py b/scripts/registry_audit.py new file mode 100644 index 0000000..8f197d2 --- /dev/null +++ b/scripts/registry_audit.py @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from datetime import date +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_REGISTRY_DIR = ROOT / "registry" +REQUIRED_PACKAGE_FIELDS = [ + "name", + "version", + "description", + "targets", + "maturity", + "owner", + "review_cadence", + "trust_level", + "license", +] + + +def display_path(path: Path, root: Path = ROOT) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def read_frontmatter(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + text = path.read_text(encoding="utf-8", errors="replace") + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + if yaml is not None: + payload = yaml.safe_load(parts[1]) or {} + return payload if isinstance(payload, dict) else {} + data: dict[str, Any] = {} + for line in parts[1].splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data + + +def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict[str, Any], str]: + candidates = [ + skill_dir / "reports" / "skill-ir.json", + skill_dir / "skill-ir" / "examples" / f"{name}.json", + skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json", + ] + for path in candidates: + payload = load_json(path) + if payload: + return payload, display_path(path, skill_dir) + return {}, "missing" + + +def license_id(skill_dir: Path) -> str: + for name in ("LICENSE", "LICENSE.md", "LICENSE.txt"): + path = skill_dir / name + if not path.exists(): + continue + text = path.read_text(encoding="utf-8", errors="replace") + if re.search(r"\bMIT License\b", text, re.IGNORECASE): + return "MIT" + for line in text.splitlines(): + value = line.strip() + if value: + return value[:80] + return "" + + +def compatibility_matrix(skill_dir: Path, declared_targets: list[str]) -> dict[str, str]: + conformance = load_json(skill_dir / "reports" / "conformance_matrix.json") + matrix = {} + for target in conformance.get("targets", []): + name = str(target.get("target", "")) + if name: + matrix[name] = str(target.get("status", "unknown")) + if "agent-skills-compatible" in declared_targets and "agent-skills" in matrix: + matrix["agent-skills-compatible"] = matrix["agent-skills"] + for target in declared_targets: + matrix.setdefault(str(target), "missing") + return matrix + + +def package_hash(skill_dir: Path) -> str: + trust = load_json(skill_dir / "reports" / "security_trust_report.json") + return str(trust.get("summary", {}).get("package_sha256", "")) + + +def package_verification(skill_dir: Path) -> dict[str, Any]: + return load_json(skill_dir / "reports" / "package_verification.json") + + +def install_simulation(skill_dir: Path) -> dict[str, Any]: + return load_json(skill_dir / "reports" / "install_simulation.json") + + +def adoption_drift(skill_dir: Path) -> dict[str, Any]: + return load_json(skill_dir / "reports" / "adoption_drift_report.json") + + +def review_waivers(skill_dir: Path) -> dict[str, Any]: + return load_json(skill_dir / "reports" / "review_waivers.json") + + +def review_annotations(skill_dir: Path) -> dict[str, Any]: + return load_json(skill_dir / "reports" / "review_annotations.json") + + +def compiled_targets(skill_dir: Path) -> dict[str, Any]: + return load_json(skill_dir / "reports" / "compiled_targets.json") + + +def package_targets(manifest: dict[str, Any], interface_doc: dict[str, Any], ir: dict[str, Any]) -> list[str]: + candidates = ( + manifest.get("target_platforms") + or ir.get("targets") + or interface_doc.get("compatibility", {}).get("adapter_targets") + or [] + ) + targets = [] + for item in candidates: + value = str(item).strip() + if value and value not in targets: + targets.append(value) + return targets + + +def build_package_metadata(skill_dir: Path, generated_at: str) -> dict[str, Any]: + frontmatter = read_frontmatter(skill_dir / "SKILL.md") + manifest = load_json(skill_dir / "manifest.json") + interface_doc = load_yaml(skill_dir / "agents" / "interface.yaml") + compatibility = interface_doc.get("compatibility", {}) + trust = compatibility.get("trust", {}) + name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name) + ir, ir_source = find_skill_ir(skill_dir, name) + verification = package_verification(skill_dir) + install = install_simulation(skill_dir) + adoption = adoption_drift(skill_dir) + waivers = review_waivers(skill_dir) + annotations = review_annotations(skill_dir) + compiled = compiled_targets(skill_dir) + archive_sha = str(verification.get("summary", {}).get("archive_sha256", "")) + checksums = {"package_sha256": package_hash(skill_dir)} + if archive_sha: + checksums["archive_sha256"] = archive_sha + description = str( + ir.get("trigger_surface", {}).get("description") + or frontmatter.get("description") + or manifest.get("description") + or "" + ) + targets = package_targets(manifest, interface_doc, ir) + package = { + "schema_version": "2.0", + "name": name, + "version": str(manifest.get("version") or frontmatter.get("version") or ""), + "description": description, + "targets": targets, + "maturity": str(manifest.get("maturity_tier") or manifest.get("lifecycle_stage") or "scaffold"), + "owner": str(manifest.get("owner") or ""), + "review_cadence": str(manifest.get("review_cadence") or ""), + "trust_level": str(trust.get("source_tier") or ""), + "license": license_id(skill_dir), + "checksums": checksums, + "compatibility": compatibility_matrix(skill_dir, targets), + "source": { + "skill_dir": display_path(skill_dir), + "skill_ir": ir_source, + "ir_schema_version": str(ir.get("schema_version") or "missing"), + "canonical_metadata": "agents/interface.yaml" if (skill_dir / "agents" / "interface.yaml").exists() else "missing", + }, + "artifacts": { + "overview_html": "reports/skill-overview.html", + "review_studio_html": "reports/review-studio.html", + "trust_report": "reports/security_trust_report.md", + "conformance_matrix": "reports/conformance_matrix.md", + "compiled_targets": "reports/compiled_targets.md" if compiled else "", + "package_verification": "reports/package_verification.md" if verification else "", + "install_simulation": "reports/install_simulation.md" if install else "", + "adoption_drift": "reports/adoption_drift_report.md" if adoption else "", + "review_waivers": "reports/review_waivers.md" if waivers else "", + "review_annotations": "reports/review_annotations.md" if annotations else "", + "package_metadata": f"registry/packages/{name}.json", + }, + "distribution": { + "archive_verified": bool(verification.get("ok")) if verification else False, + "archive_sha256": archive_sha, + "package_verification": "reports/package_verification.json" if verification else "", + "install_simulated": bool(install.get("ok")) if install else False, + "install_simulation": "reports/install_simulation.json" if install else "", + }, + "generated_at": generated_at, + } + return package + + +def audit_package(package: dict[str, Any], skill_dir: Path) -> tuple[list[str], list[str]]: + failures = [] + warnings = [] + for field in REQUIRED_PACKAGE_FIELDS: + if not package.get(field): + failures.append(f"Missing package metadata field: {field}") + if not package.get("checksums", {}).get("package_sha256"): + failures.append("Missing package checksum: checksums.package_sha256") + if package.get("source", {}).get("ir_schema_version") != "2.0.0": + failures.append("Missing valid Skill IR source for registry package") + compatibility = package.get("compatibility", {}) + for target in package.get("targets", []): + if compatibility.get(target) not in {"pass", "warn"}: + failures.append(f"Compatibility is not passing for target: {target}") + for rel in ( + "reports/skill-overview.html", + "reports/review-studio.html", + "reports/compiled_targets.md", + "reports/security_trust_report.md", + "reports/review_waivers.md", + "reports/review_annotations.md", + ): + if not (skill_dir / rel).exists(): + warnings.append(f"Recommended registry evidence is missing: {rel}") + verification = package.get("distribution", {}) + if verification.get("package_verification") and not verification.get("archive_verified"): + failures.append("Package verification report exists but archive verification did not pass") + if verification.get("install_simulation") and not verification.get("install_simulated"): + failures.append("Install simulation report exists but install simulation did not pass") + return failures, warnings + + +def build_index(package: dict[str, Any], generated_at: str) -> dict[str, Any]: + return { + "schema_version": "2.0", + "generated_at": generated_at, + "package_count": 1, + "packages": [ + { + "name": package["name"], + "version": package["version"], + "maturity": package["maturity"], + "owner": package["owner"], + "targets": package["targets"], + "package_metadata": package["artifacts"]["package_metadata"], + "package_sha256": package["checksums"]["package_sha256"], + } + ], + } + + +def render_markdown(report: dict[str, Any]) -> str: + package = report["package"] + lines = [ + "# Registry Audit", + "", + f"- OK: `{report['ok']}`", + f"- Package: `{package['name']}`", + f"- Version: `{package['version']}`", + f"- Maturity: `{package['maturity']}`", + f"- Owner: `{package['owner']}`", + f"- License: `{package['license']}`", + f"- Package SHA256: `{package['checksums']['package_sha256']}`", + f"- Archive SHA256: `{package['checksums'].get('archive_sha256', 'n/a')}`", + f"- Install simulated: `{package.get('distribution', {}).get('install_simulated', False)}`", + "", + "## Compatibility", + "", + "| Target | Status |", + "| --- | --- |", + ] + for target, status in package["compatibility"].items(): + lines.append(f"| `{target}` | `{status}` |") + lines.extend(["", "## Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) + lines.extend(["", "## Warnings", ""]) + lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"]) + lines.extend(["", "## Artifacts", ""]) + for key, value in report["artifacts"].items(): + lines.append(f"- {key}: `{value}`") + return "\n".join(lines) + "\n" + + +def run_registry_audit( + skill_dir: Path, + registry_dir: Path, + output_json: Path, + output_md: Path, + generated_at: str, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + registry_dir = registry_dir.resolve() + package = build_package_metadata(skill_dir, generated_at) + failures, warnings = audit_package(package, skill_dir) + index = build_index(package, generated_at) + + package_dir = registry_dir / "packages" + package_path = package_dir / f"{package['name']}.json" + index_path = registry_dir / "index.json" + package_dir.mkdir(parents=True, exist_ok=True) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + package_path.write_text(json.dumps(package, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + index_path.write_text(json.dumps(index, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + + report = { + "ok": not failures, + "schema_version": "2.0", + "skill_dir": display_path(skill_dir), + "registry_dir": display_path(registry_dir), + "package": package, + "index": index, + "failures": failures, + "warnings": warnings, + "artifacts": { + "index": display_path(index_path), + "package": display_path(package_path), + "json": display_path(output_json), + "markdown": display_path(output_md), + }, + } + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build and audit Skill OS registry package metadata.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--registry-dir", default=str(DEFAULT_REGISTRY_DIR)) + parser.add_argument("--output-json", default="reports/registry_audit.json") + parser.add_argument("--output-md", default="reports/registry_audit.md") + parser.add_argument("--generated-at", default=str(date.today())) + args = parser.parse_args() + + payload = run_registry_audit( + Path(args.skill_dir), + Path(args.registry_dir), + Path(args.output_json), + Path(args.output_md), + args.generated_at, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_adoption_drift_report.py b/scripts/render_adoption_drift_report.py new file mode 100644 index 0000000..9ba81aa --- /dev/null +++ b/scripts/render_adoption_drift_report.py @@ -0,0 +1,441 @@ +#!/usr/bin/env python3 +import argparse +import json +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent + +ALLOWED_EVENTS = {"skill_activation", "skill_output", "script_run", "review_event"} +ALLOWED_ACTIVATION_TYPES = {"implicit", "explicit", "manual", "unknown"} +ALLOWED_OUTCOMES = {"accepted", "edited", "rejected", "missed", "failed", "reviewed", "unknown"} +ALLOWED_FAILURE_TYPES = { + "none", + "wrong_trigger", + "under_trigger", + "bad_output", + "missing_resource", + "script_error", + "review_overdue", +} +ALLOWED_FIELDS = { + "event", + "skill", + "version", + "activation_type", + "outcome", + "failure_type", + "timestamp", +} +SENSITIVE_FIELDS = { + "prompt", + "content", + "input", + "inputs", + "output", + "outputs", + "transcript", + "message", + "messages", + "note", + "text", + "raw", +} + + +def display_path(path: Path, root: Path = ROOT) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def read_frontmatter(path: Path) -> dict[str, str]: + if not path.exists(): + return {} + text = path.read_text(encoding="utf-8", errors="replace") + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + data: dict[str, str] = {} + for line in parts[1].splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data + + +def skill_defaults(skill_dir: Path) -> dict[str, str]: + manifest = load_json(skill_dir / "manifest.json") + frontmatter = read_frontmatter(skill_dir / "SKILL.md") + return { + "skill": str(manifest.get("name") or frontmatter.get("name") or skill_dir.name), + "version": str(manifest.get("version") or frontmatter.get("version") or "0.0.0"), + } + + +def utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def normalize_event(raw: dict[str, Any], defaults: dict[str, str], line_label: str) -> tuple[dict[str, str] | None, list[str]]: + failures: list[str] = [] + raw_keys = set(raw.keys()) + sensitive = sorted(raw_keys & SENSITIVE_FIELDS) + if sensitive: + failures.append(f"{line_label}: raw content fields are not allowed in telemetry: {', '.join(sensitive)}") + unknown = sorted(raw_keys - ALLOWED_FIELDS - SENSITIVE_FIELDS) + if unknown: + failures.append(f"{line_label}: unknown telemetry fields are blocked by the metadata-only contract: {', '.join(unknown)}") + + event = str(raw.get("event") or "skill_activation") + activation_type = str(raw.get("activation_type") or "unknown") + outcome = str(raw.get("outcome") or "unknown") + failure_type = str(raw.get("failure_type") or "none") + timestamp = str(raw.get("timestamp") or utc_now()) + skill = str(raw.get("skill") or defaults["skill"]) + version = str(raw.get("version") or defaults["version"]) + + if event not in ALLOWED_EVENTS: + failures.append(f"{line_label}: unsupported event `{event}`") + if activation_type not in ALLOWED_ACTIVATION_TYPES: + failures.append(f"{line_label}: unsupported activation_type `{activation_type}`") + if outcome not in ALLOWED_OUTCOMES: + failures.append(f"{line_label}: unsupported outcome `{outcome}`") + if failure_type not in ALLOWED_FAILURE_TYPES: + failures.append(f"{line_label}: unsupported failure_type `{failure_type}`") + + if failures: + return None, failures + return { + "event": event, + "skill": skill, + "version": version, + "activation_type": activation_type, + "outcome": outcome, + "failure_type": failure_type, + "timestamp": timestamp, + }, [] + + +def load_events(path: Path, defaults: dict[str, str]) -> tuple[list[dict[str, str]], list[str]]: + if not path.exists(): + return [], [] + events: list[dict[str, str]] = [] + failures: list[str] = [] + for index, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + if not line.strip(): + continue + try: + raw = json.loads(line) + except json.JSONDecodeError as exc: + failures.append(f"line {index}: invalid JSONL event: {exc.msg}") + continue + if not isinstance(raw, dict): + failures.append(f"line {index}: telemetry event must be a JSON object") + continue + event, event_failures = normalize_event(raw, defaults, f"line {index}") + failures.extend(event_failures) + if event: + events.append(event) + return events, failures + + +def append_event(path: Path, event: dict[str, str]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(json.dumps(event, ensure_ascii=False, sort_keys=True) + "\n") + + +def adoption_by_skill(events: list[dict[str, str]]) -> list[dict[str, Any]]: + grouped: dict[str, Counter[str]] = defaultdict(Counter) + for event in events: + grouped[event["skill"]][event["outcome"]] += 1 + rows = [] + for skill, counts in sorted(grouped.items()): + total = sum(counts.values()) + adopted = counts["accepted"] + counts["edited"] + rows.append( + { + "skill": skill, + "events": total, + "accepted": counts["accepted"], + "edited": counts["edited"], + "rejected": counts["rejected"], + "missed": counts["missed"], + "adoption_rate": round(adopted / total * 100, 1) if total else 0, + } + ) + return rows + + +def atlas_review_overdue(skill_dir: Path) -> int: + atlas = load_json(skill_dir / "reports" / "skill_atlas.json") + stale = atlas.get("stale_skills", []) if isinstance(atlas.get("stale_skills"), list) else [] + return sum( + 1 + for item in stale + if item.get("actionable", True) and "overdue" in str(item.get("reason", "")).lower() + ) + + +def next_candidates(summary: dict[str, Any]) -> list[dict[str, str]]: + candidates: list[dict[str, str]] = [] + if summary["event_count"] == 0: + candidates.append( + { + "signal": "no telemetry", + "recommendation": "Start with a small metadata-only sample before using telemetry for release decisions.", + } + ) + if summary["missed_trigger_count"]: + candidates.append( + { + "signal": "missed trigger", + "recommendation": "Add missed prompts to trigger eval and tighten should-trigger examples.", + } + ) + if summary["wrong_trigger_count"]: + candidates.append( + { + "signal": "wrong trigger", + "recommendation": "Add near-neighbor should-not-trigger cases and clarify exclusions.", + } + ) + if summary["bad_output_count"]: + candidates.append( + { + "signal": "bad output", + "recommendation": "Convert the failed output shape into an Output Eval assertion and failure taxonomy entry.", + } + ) + if summary["script_error_count"]: + candidates.append( + { + "signal": "script error", + "recommendation": "Add non-interactive script smoke tests and clearer error paths.", + } + ) + if summary["review_overdue_count"]: + candidates.append( + { + "signal": "review overdue", + "recommendation": "Prioritize stale governed skills in Skill Atlas before creating more assets.", + } + ) + return candidates[:6] + + +def summarize(events: list[dict[str, str]], review_overdue_count: int) -> dict[str, Any]: + outcomes = Counter(event["outcome"] for event in events) + failures = Counter(event["failure_type"] for event in events if event["failure_type"] != "none") + event_types = Counter(event["event"] for event in events) + adopted = outcomes["accepted"] + outcomes["edited"] + event_count = len(events) + missed_trigger = outcomes["missed"] + failures["under_trigger"] + bad_output = failures["bad_output"] + script_error = failures["script_error"] + wrong_trigger = failures["wrong_trigger"] + risk_band = "no-data" + if event_count: + risk_points = missed_trigger + bad_output + script_error + wrong_trigger + review_overdue_count + if risk_points >= 4: + risk_band = "high" + elif risk_points: + risk_band = "medium" + else: + risk_band = "low" + return { + "event_count": event_count, + "activation_count": event_types["skill_activation"], + "accepted_count": outcomes["accepted"], + "edited_count": outcomes["edited"], + "rejected_count": outcomes["rejected"], + "missed_count": outcomes["missed"], + "failed_count": outcomes["failed"], + "adoption_rate": round(adopted / event_count * 100, 1) if event_count else 0, + "missed_trigger_count": missed_trigger, + "wrong_trigger_count": wrong_trigger, + "bad_output_count": bad_output, + "script_error_count": script_error, + "missing_resource_count": failures["missing_resource"], + "review_overdue_count": review_overdue_count + failures["review_overdue"], + "risk_band": risk_band, + "event_types": dict(sorted(event_types.items())), + "failure_types": dict(sorted(failures.items())), + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Adoption And Drift Report", + "", + "Local-first, metadata-only telemetry for skill operations. Raw prompts, outputs, transcripts, and notes are not allowed in the event stream.", + "", + "## Summary", + "", + f"- Events: `{summary['event_count']}`", + f"- Activation events: `{summary['activation_count']}`", + f"- Adoption rate: `{summary['adoption_rate']}`", + f"- Missed trigger signals: `{summary['missed_trigger_count']}`", + f"- Bad output signals: `{summary['bad_output_count']}`", + f"- Script error signals: `{summary['script_error_count']}`", + f"- Review overdue signals: `{summary['review_overdue_count']}`", + f"- Risk band: `{summary['risk_band']}`", + "", + "## Privacy Contract", + "", + "- Storage is local-first.", + "- Events are metadata-only.", + "- Raw user prompts, model outputs, transcripts, notes, and messages are blocked.", + "- Distributed packages should include this aggregate report, not raw `reports/telemetry_events.jsonl`.", + "", + "## Adoption By Skill", + "", + "| Skill | Events | Accepted | Edited | Rejected | Missed | Adoption Rate |", + "| --- | ---: | ---: | ---: | ---: | ---: | ---: |", + ] + for row in report["adoption_by_skill"]: + lines.append( + f"| `{row['skill']}` | {row['events']} | {row['accepted']} | {row['edited']} | " + f"{row['rejected']} | {row['missed']} | {row['adoption_rate']} |" + ) + if not report["adoption_by_skill"]: + lines.append("| `none` | 0 | 0 | 0 | 0 | 0 | 0 |") + lines.extend(["", "## Next Iteration Candidates", ""]) + for item in report["next_iteration_candidates"]: + lines.append(f"- `{item['signal']}`: {item['recommendation']}") + if not report["next_iteration_candidates"]: + lines.append("- No telemetry-driven iteration candidate yet.") + lines.extend(["", "## Recent Metadata Events", ""]) + for event in report["recent_events"]: + lines.append( + f"- `{event['timestamp']}` `{event['skill']}` event=`{event['event']}` " + f"activation=`{event['activation_type']}` outcome=`{event['outcome']}` failure=`{event['failure_type']}`" + ) + if not report["recent_events"]: + lines.append("- No metadata events captured yet.") + if report["failures"]: + lines.extend(["", "## Validation Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]]) + return "\n".join(lines) + "\n" + + +def render_report( + skill_dir: Path, + events_jsonl: Path | None = None, + output_json: Path | None = None, + output_md: Path | None = None, + generated_at: str | None = None, + record_event: dict[str, Any] | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + defaults = skill_defaults(skill_dir) + events_jsonl = events_jsonl or reports_dir / "telemetry_events.jsonl" + output_json = output_json or reports_dir / "adoption_drift_report.json" + output_md = output_md or reports_dir / "adoption_drift_report.md" + generated_at = generated_at or utc_now() + failures: list[str] = [] + + if record_event is not None: + event, event_failures = normalize_event(record_event, defaults, "record") + failures.extend(event_failures) + if event: + append_event(events_jsonl, event) + + events, load_failures = load_events(events_jsonl, defaults) + failures.extend(load_failures) + summary = summarize(events, atlas_review_overdue(skill_dir)) + report = { + "ok": not failures, + "schema_version": "2.0", + "generated_at": generated_at, + "skill_dir": display_path(skill_dir), + "privacy_contract": { + "storage": "local-first", + "event_scope": "metadata-only", + "raw_content_allowed": False, + "raw_event_log_packaged": False, + "blocked_fields": sorted(SENSITIVE_FIELDS), + }, + "summary": summary, + "adoption_by_skill": adoption_by_skill(events), + "next_iteration_candidates": next_candidates(summary), + "recent_events": events[-10:], + "failures": failures, + "artifacts": { + "events_jsonl": display_path(events_jsonl), + "json": display_path(output_json), + "markdown": display_path(output_md), + }, + } + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render local-first adoption and drift telemetry for a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--events-jsonl") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--generated-at") + parser.add_argument("--record-event", choices=sorted(ALLOWED_EVENTS)) + parser.add_argument("--activation-type", choices=sorted(ALLOWED_ACTIVATION_TYPES), default="unknown") + parser.add_argument("--outcome", choices=sorted(ALLOWED_OUTCOMES), default="unknown") + parser.add_argument("--failure-type", choices=sorted(ALLOWED_FAILURE_TYPES), default="none") + parser.add_argument("--timestamp") + parser.add_argument("--skill-name") + parser.add_argument("--version") + args = parser.parse_args() + + record_event = None + if args.record_event: + record_event = { + "event": args.record_event, + "activation_type": args.activation_type, + "outcome": args.outcome, + "failure_type": args.failure_type, + } + if args.timestamp: + record_event["timestamp"] = args.timestamp + if args.skill_name: + record_event["skill"] = args.skill_name + if args.version: + record_event["version"] = args.version + + report = render_report( + Path(args.skill_dir), + events_jsonl=Path(args.events_jsonl).resolve() if args.events_jsonl else None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + output_md=Path(args.output_md).resolve() if args.output_md else None, + generated_at=args.generated_at, + record_event=record_event, + ) + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_context_reports.py b/scripts/render_context_reports.py index fc82285..ea57191 100644 --- a/scripts/render_context_reports.py +++ b/scripts/render_context_reports.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import argparse import json from pathlib import Path @@ -25,7 +26,18 @@ TARGETS = [ ] +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Render context budget reports for root and example skills.") + parser.add_argument( + "--generated-at", + default="2026-03-31", + help="Report date written to context_budget_summary.json.", + ) + return parser.parse_args() + + def main() -> None: + args = parse_args() rows = [] for target in TARGETS: report = analyze_skill(target["path"]) @@ -48,7 +60,7 @@ def main() -> None: ) summary = { - "generated_at": "2026-03-31", + "generated_at": args.generated_at, "targets": rows, } (ROOT / "reports").mkdir(parents=True, exist_ok=True) diff --git a/scripts/render_review_annotations.py b/scripts/render_review_annotations.py new file mode 100644 index 0000000..58dbe68 --- /dev/null +++ b/scripts/render_review_annotations.py @@ -0,0 +1,399 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_SOURCE = ROOT / "reports" / "review_annotations_input.json" +DEFAULT_OUTPUT_JSON = ROOT / "reports" / "review_annotations.json" +DEFAULT_OUTPUT_MD = ROOT / "reports" / "review_annotations.md" +VALID_GATES = { + "intent-canvas", + "trigger-lab", + "output-lab", + "context-budget", + "runtime-matrix", + "trust-report", + "permission-gates", + "permission-runtime", + "skill-atlas", + "operations-loop", + "review-waivers", + "registry-audit", + "release-notes", +} +VALID_SEVERITIES = {"info", "note", "warning", "blocker"} +VALID_STATUSES = {"open", "resolved", "deferred"} + + +def display_path(root: Path, path: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> tuple[dict[str, Any], list[str]]: + if not path.exists(): + return {"schema_version": "1.0", "annotations": []}, [] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError as exc: + return {"schema_version": "1.0", "annotations": []}, [f"Invalid annotation JSON {display_path(ROOT, path)}: {exc}"] + if not isinstance(payload, dict): + return {"schema_version": "1.0", "annotations": []}, [f"Annotation file root must be an object: {display_path(ROOT, path)}"] + if not isinstance(payload.get("annotations", []), list): + return {"schema_version": "1.0", "annotations": []}, ["annotations must be a list"] + return payload, [] + + +def template_payload() -> dict[str, Any]: + return { + "schema_version": "1.0", + "review_contract": { + "gate_key": "One Review Studio gate key, such as output-lab or trust-report.", + "target_path": "Relative path to the source or report being annotated.", + "line": "Optional positive integer line number.", + "severity": "info | note | warning | blocker.", + "status": "open | resolved | deferred.", + "body": "Reviewer-visible annotation text.", + }, + "annotations": [], + } + + +def annotation_id(annotation: dict[str, Any]) -> str: + if annotation.get("id"): + return str(annotation["id"]) + raw = "|".join( + [ + str(annotation.get("gate_key", "")), + str(annotation.get("target_path", "")), + str(annotation.get("line", "")), + str(annotation.get("body", "")), + ] + ) + return "ann-" + hashlib.sha256(raw.encode("utf-8")).hexdigest()[:12] + + +def source_excerpt(path: Path, line: int | None) -> str: + if line is None or not path.exists() or not path.is_file(): + return "" + try: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return "" + if line < 1 or line > len(lines): + return "" + return lines[line - 1].strip()[:220] + + +def normalize_annotation(skill_dir: Path, raw: dict[str, Any], index: int) -> tuple[dict[str, Any], list[str]]: + failures: list[str] = [] + if not isinstance(raw, dict): + return {}, [f"annotation #{index} must be an object"] + + gate_key = str(raw.get("gate_key", "")).strip() + if gate_key not in VALID_GATES: + failures.append(f"{annotation_id(raw)}: invalid gate_key {gate_key!r}") + + severity = str(raw.get("severity", "note")).strip().lower() + if severity not in VALID_SEVERITIES: + failures.append(f"{annotation_id(raw)}: invalid severity {severity!r}") + + status = str(raw.get("status", "open")).strip().lower() + if status not in VALID_STATUSES: + failures.append(f"{annotation_id(raw)}: invalid status {status!r}") + + body = str(raw.get("body", "")).strip() + if not body: + failures.append(f"{annotation_id(raw)}: body is required") + + target_path = str(raw.get("target_path", "")).strip() + if not target_path: + failures.append(f"{annotation_id(raw)}: target_path is required") + rel = Path(target_path) + if rel.is_absolute(): + failures.append(f"{annotation_id(raw)}: target_path must be relative") + target = skill_dir / "__invalid_absolute_path__" + else: + target = (skill_dir / rel).resolve() + try: + target.relative_to(skill_dir.resolve()) + except ValueError: + failures.append(f"{annotation_id(raw)}: target_path escapes skill directory") + target = skill_dir / "__invalid_escaped_path__" + + line_value = raw.get("line", None) + line: int | None + if line_value is None or line_value == "": + line = None + else: + try: + line = int(line_value) + except (TypeError, ValueError): + line = None + failures.append(f"{annotation_id(raw)}: line must be a positive integer") + else: + if line < 1: + failures.append(f"{annotation_id(raw)}: line must be a positive integer") + + normalized = { + "id": annotation_id(raw), + "gate_key": gate_key, + "target_path": target_path, + "line": line, + "severity": severity, + "status": status, + "reviewer": str(raw.get("reviewer", "")).strip(), + "created_at": str(raw.get("created_at", "")).strip(), + "body": body, + "suggested_action": str(raw.get("suggested_action", "")).strip(), + "evidence": str(raw.get("evidence", "")).strip(), + "target_exists": target.exists(), + "source_excerpt": source_excerpt(target, line), + } + return normalized, failures + + +def append_annotation(source: dict[str, Any], args: argparse.Namespace) -> dict[str, Any]: + annotations = list(source.get("annotations", [])) + raw = { + "id": args.annotation_id, + "gate_key": args.gate_key, + "target_path": args.target_path, + "line": args.line, + "severity": args.severity, + "status": args.status, + "reviewer": args.reviewer, + "created_at": args.created_at or str(date.today()), + "body": args.body, + "suggested_action": args.suggested_action, + "evidence": args.evidence, + } + annotations.append({key: value for key, value in raw.items() if value is not None and value != ""}) + updated = dict(source) + updated["annotations"] = annotations + updated.setdefault("schema_version", "1.0") + return updated + + +def build_summary(annotations: list[dict[str, Any]], failures: list[str]) -> dict[str, Any]: + open_items = [item for item in annotations if item["status"] == "open"] + resolved_items = [item for item in annotations if item["status"] == "resolved"] + deferred_items = [item for item in annotations if item["status"] == "deferred"] + open_blockers = [item for item in open_items if item["severity"] == "blocker"] + open_warnings = [item for item in open_items if item["severity"] == "warning"] + return { + "annotation_count": len(annotations), + "open_count": len(open_items), + "resolved_count": len(resolved_items), + "deferred_count": len(deferred_items), + "open_blocker_count": len(open_blockers), + "open_warning_count": len(open_warnings), + "linked_gate_count": len({item["gate_key"] for item in annotations if item["gate_key"]}), + "target_missing_count": sum(1 for item in annotations if not item["target_exists"]), + "failure_count": len(failures), + } + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + "# Review Annotations", + "", + "This report renders reviewer annotations attached to Review Studio gates and source/report paths.", + "", + f"- Annotations: `{summary['annotation_count']}`", + f"- Open: `{summary['open_count']}`", + f"- Resolved: `{summary['resolved_count']}`", + f"- Deferred: `{summary['deferred_count']}`", + f"- Open blockers: `{summary['open_blocker_count']}`", + f"- Open warnings: `{summary['open_warning_count']}`", + "", + ] + if not payload["annotations"]: + lines.extend(["No reviewer annotations recorded yet.", ""]) + else: + lines.extend( + [ + "## Annotation Ledger", + "", + "| ID | Gate | Severity | Status | Target | Reviewer | Note |", + "| --- | --- | --- | --- | --- | --- | --- |", + ] + ) + for item in payload["annotations"]: + line = f":{item['line']}" if item.get("line") else "" + note = item["body"].replace("|", "\\|") + lines.append( + f"| {item['id']} | {item['gate_key']} | {item['severity']} | {item['status']} | " + f"{item['target_path']}{line} | {item.get('reviewer', '')} | {note} |" + ) + if item.get("source_excerpt"): + excerpt = str(item["source_excerpt"]).replace("|", "\\|") + lines.append(f"| | | | | | excerpt | `{excerpt}` |") + if payload.get("failures"): + lines.extend(["", "## Failures", ""]) + for failure in payload["failures"]: + lines.append(f"- {failure}") + lines.extend( + [ + "", + "## Review Rule", + "", + "- Use annotations for reviewer comments tied to a gate or source line.", + "- Use waivers only for explicit acceptance of warning-level release risk.", + "- Open blocker annotations should block a release decision until resolved or deferred with rationale.", + ] + ) + return "\n".join(lines).strip() + "\n" + + +def render_review_annotations( + skill_dir: Path, + source_json: Path, + output_json: Path, + output_md: Path, + args: argparse.Namespace, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + source, failures = load_json(source_json) + template_written = False + should_write_source = False + if args.write_template and not source_json.exists(): + source_json.parent.mkdir(parents=True, exist_ok=True) + source_json.write_text(json.dumps(template_payload(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + source = template_payload() + template_written = True + if args.add_annotation: + source = append_annotation(source, args) + should_write_source = True + + annotations: list[dict[str, Any]] = [] + seen = set() + for index, raw in enumerate(source.get("annotations", []), start=1): + normalized, item_failures = normalize_annotation(skill_dir, raw, index) + failures.extend(item_failures) + if normalized: + if normalized["id"] in seen: + failures.append(f"{normalized['id']}: duplicate annotation id") + seen.add(normalized["id"]) + annotations.append(normalized) + + summary = build_summary(annotations, failures) + if should_write_source and not failures: + source_json.parent.mkdir(parents=True, exist_ok=True) + source_json.write_text(json.dumps(source, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + payload = { + "schema_version": "1.0", + "ok": not failures, + "skill_dir": display_path(ROOT, skill_dir), + "source": display_path(skill_dir, source_json), + "summary": summary, + "annotations": annotations, + "failures": failures, + "template_written": template_written, + "artifacts": { + "json": display_path(skill_dir, output_json), + "markdown": display_path(skill_dir, output_md), + }, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def render_report( + skill_dir: Path, + annotations_json: Path | None = None, + output_json: Path | None = None, + output_md: Path | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + args = argparse.Namespace( + write_template=False, + add_annotation=False, + annotation_id=None, + gate_key=None, + target_path=None, + line=None, + severity="note", + status="open", + reviewer=None, + created_at=None, + body=None, + suggested_action=None, + evidence=None, + ) + return render_review_annotations( + skill_dir, + annotations_json or skill_dir / "reports" / "review_annotations_input.json", + output_json or skill_dir / "reports" / "review_annotations.json", + output_md or skill_dir / "reports" / "review_annotations.md", + args, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render reviewer annotations for Review Studio gates and source paths.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--annotations-json") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--write-template", action="store_true") + parser.add_argument("--add-annotation", action="store_true") + parser.add_argument("--annotation-id") + parser.add_argument("--gate-key", choices=sorted(VALID_GATES)) + parser.add_argument("--target-path") + parser.add_argument("--line", type=int) + parser.add_argument("--severity", choices=sorted(VALID_SEVERITIES), default="note") + parser.add_argument("--status", choices=sorted(VALID_STATUSES), default="open") + parser.add_argument("--reviewer") + parser.add_argument("--created-at") + parser.add_argument("--body") + parser.add_argument("--suggested-action") + parser.add_argument("--evidence") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + annotations_json = Path(args.annotations_json).resolve() if args.annotations_json else skill_dir / "reports" / "review_annotations_input.json" + output_json = Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "review_annotations.json" + output_md = Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "review_annotations.md" + + if args.add_annotation: + missing = [name for name in ("gate_key", "target_path", "body") if not getattr(args, name)] + if missing: + payload = { + "schema_version": "1.0", + "ok": False, + "skill_dir": display_path(ROOT, skill_dir), + "source": display_path(skill_dir, annotations_json), + "summary": build_summary([], missing), + "annotations": [], + "failures": [f"--add-annotation requires {', '.join(missing)}"], + "template_written": False, + "artifacts": { + "json": display_path(skill_dir, output_json), + "markdown": display_path(skill_dir, output_md), + }, + } + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(2) + + payload = render_review_annotations(skill_dir, annotations_json, output_json, output_md, args) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index 6ce6f77..1c1b7cf 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -11,6 +11,13 @@ try: except ImportError: # pragma: no cover yaml = None +try: + from trust_check import permission_governance_status as compute_permission_governance_status + from trust_check import script_inventory as trust_script_inventory +except ImportError: # pragma: no cover + compute_permission_governance_status = None + trust_script_inventory = None + ROOT = Path(__file__).resolve().parent.parent @@ -72,6 +79,48 @@ def report_link(output_html: Path, skill_dir: Path, rel_path: str) -> str: return link_from(output_html, skill_dir / rel_path) +def find_line(path: Path, patterns: list[str] | None = None) -> int | None: + if not path.exists(): + return None + if not patterns: + return 1 + try: + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + except OSError: + return 1 + for pattern in patterns: + for index, line in enumerate(lines, start=1): + if pattern in line: + return index + return 1 + + +def source_refs( + skill_dir: Path, + output_html: Path, + specs: list[dict[str, Any]], +) -> list[dict[str, Any]]: + refs: list[dict[str, Any]] = [] + for spec in specs: + rel_path = str(spec.get("path", "")).strip() + if not rel_path: + continue + path = skill_dir / rel_path + exists = path.exists() + line = find_line(path, spec.get("patterns", [])) + refs.append( + { + "path": rel_path, + "label": str(spec.get("label", rel_path)), + "kind": str(spec.get("kind", "source")), + "line": line, + "exists": exists, + "link": link_from(output_html, path) if exists else "", + } + ) + return refs + + def gate(key: str, label: str, status: str, detail: str, evidence: str, link: str = "") -> dict[str, str]: return { "key": key, @@ -111,6 +160,16 @@ def min_output_cases(maturity: str) -> int: return 1 +def fallback_permission_governance(skill_dir: Path) -> dict[str, Any]: + if compute_permission_governance_status is None or trust_script_inventory is None: + return {} + try: + scripts = trust_script_inventory(skill_dir) + return compute_permission_governance_status(skill_dir, scripts) + except Exception: + return {} + + def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: overview = data["overview"] maturity = target_maturity(skill_dir, overview) @@ -152,14 +211,37 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An ) output = data["output_quality"] + output_execution = data["output_execution"] + output_blind = data["output_blind_review"] + output_review = data["output_review_adjudication"] output_summary = output.get("summary", {}) + output_execution_summary = output_execution.get("summary", {}) + output_blind_summary = output_blind.get("summary", {}) + output_review_summary = output_review.get("summary", {}) required_cases = min_output_cases(maturity) case_count = int(output_summary.get("case_count", 0) or 0) file_backed = int(output_summary.get("file_backed_case_count", 0) or 0) near_neighbor = int(output_summary.get("near_neighbor_case_count", 0) or 0) boundary = int(output_summary.get("boundary_case_count", 0) or 0) - output_blocked = not output.get("ok", False) or not output_summary.get("gate_pass", False) or case_count < required_cases - output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 + blind_pair_count = int(output_blind_summary.get("pair_count", 0) or 0) + execution_variant_count = int(output_execution_summary.get("variant_run_count", 0) or 0) + execution_command_count = int(output_execution_summary.get("command_executed_count", 0) or 0) + execution_model_count = int(output_execution_summary.get("model_executed_count", 0) or 0) + execution_recorded_count = int(output_execution_summary.get("recorded_fixture_count", 0) or 0) + review_pair_count = int(output_review_summary.get("pair_count", 0) or 0) + review_judgment_count = int(output_review_summary.get("judgment_count", 0) or 0) + review_invalid_count = int(output_review_summary.get("invalid_decision_count", 0) or 0) + blind_missing = maturity in {"production", "library", "governed"} and (not output_blind or blind_pair_count < case_count) + execution_failed = bool(output_execution) and (not output_execution.get("ok", True) or int(output_execution_summary.get("failure_count", 0) or 0) > 0) + review_invalid = bool(output_review) and (not output_review.get("ok", True) or review_invalid_count > 0) + output_blocked = ( + not output.get("ok", False) + or not output_summary.get("gate_pass", False) + or case_count < required_cases + or execution_failed + or review_invalid + ) + output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing if not output: output_status = "warn" output_detail = "output eval scorecard is missing; generate it before production review" @@ -167,7 +249,15 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An output_status = "block" if output_blocked else ("warn" if output_warn else "pass") output_detail = ( f"{case_count}/{required_cases} cases; with-skill {output_summary.get('with_skill_pass_rate', 0)}; " - f"baseline {output_summary.get('baseline_pass_rate', 0)}; file-backed {file_backed}; near-neighbor {near_neighbor}" + f"baseline {output_summary.get('baseline_pass_rate', 0)}; file-backed {file_backed}; near-neighbor {near_neighbor}; " + f"blind A/B {blind_pair_count}" + + ( + f"; exec {execution_variant_count}; command {execution_command_count}; " + f"model {execution_model_count}; recorded {execution_recorded_count}" + if output_execution + else "" + ) + + (f"; reviewed {review_judgment_count}/{review_pair_count}" if output_review else "") ) gates.append( gate( @@ -232,7 +322,8 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An trust_detail = ( f"{trust_summary.get('secret_findings', 0)} secrets; " f"{trust_summary.get('script_count', 0)} scripts; " - f"{trust_summary.get('network_script_count', 0)} network-capable scripts" + f"{trust_summary.get('network_script_count', 0)} network-capable scripts; " + f"{trust_summary.get('help_smoke_failed_count', 0)} help smoke failures" ) gates.append( gate( @@ -245,19 +336,87 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An ) ) + permission_governance = trust.get("permission_governance", {}) if isinstance(trust.get("permission_governance", {}), dict) else {} + if trust and not permission_governance: + permission_governance = fallback_permission_governance(skill_dir) + if not trust: + permission_status = "warn" + permission_detail = "permission governance evidence is missing because trust report is missing" + elif not permission_governance: + permission_status = "warn" + permission_detail = "permission governance evidence is missing from trust report" + else: + required = int(permission_governance.get("required_count", 0) or 0) + approved = int(permission_governance.get("approval_count", 0) or 0) + gaps = ( + int(permission_governance.get("missing_count", 0) or 0) + + int(permission_governance.get("invalid_count", 0) or 0) + + int(permission_governance.get("expired_count", 0) or 0) + ) + if gaps: + permission_status = "block" if maturity == "governed" else "warn" + else: + permission_status = "pass" + required_names = ", ".join(permission_governance.get("required_capabilities", []) or []) or "none" + permission_detail = f"{approved}/{required} permissions approved; gaps {gaps}; required {required_names}" + gates.append( + gate( + "permission-gates", + "权限批准", + permission_status, + permission_detail, + "reports/security_trust_report.json + security/permission_policy.json", + report_link(output_html, skill_dir, "security/permission_policy.md"), + ) + ) + + runtime_permissions = data["runtime_permissions"] + runtime_permissions_summary = runtime_permissions.get("summary", {}) + if not runtime_permissions: + runtime_permission_status = "block" if maturity == "governed" else "warn" + runtime_permission_detail = "runtime permission probe report is missing" + elif runtime_permissions.get("failures"): + runtime_permission_status = "block" + runtime_permission_detail = f"{runtime_permissions_summary.get('failure_count', len(runtime_permissions.get('failures', [])))} runtime permission probe failures" + else: + runtime_permission_status = "pass" + runtime_permission_detail = ( + f"{runtime_permissions_summary.get('pass_count', 0)}/{runtime_permissions_summary.get('target_count', 0)} targets probed; " + f"native {runtime_permissions_summary.get('native_enforcement_count', 0)}; " + f"metadata fallback {runtime_permissions_summary.get('metadata_fallback_count', 0)}; " + f"residual risks {runtime_permissions_summary.get('residual_risk_count', 0)}" + ) + gates.append( + gate( + "permission-runtime", + "权限探针", + runtime_permission_status, + runtime_permission_detail, + "reports/runtime_permission_probes.json", + report_link(output_html, skill_dir, "reports/runtime_permission_probes.md"), + ) + ) + atlas = data["atlas"] atlas_summary = atlas.get("summary", {}) - atlas_issues = int(atlas_summary.get("route_collision_count", 0) or 0) + int(atlas_summary.get("owner_gap_count", 0) or 0) + actionable_route_collisions = int( + atlas_summary.get("actionable_route_collision_count", atlas_summary.get("route_collision_count", 0)) or 0 + ) + actionable_owner_gaps = int(atlas_summary.get("actionable_owner_gap_count", atlas_summary.get("owner_gap_count", 0)) or 0) + actionable_stale = int(atlas_summary.get("actionable_stale_count", atlas_summary.get("stale_count", 0)) or 0) + atlas_issues = actionable_route_collisions + actionable_owner_gaps + actionable_stale if not atlas: atlas_status = "warn" atlas_detail = "skill atlas is missing; portfolio-level conflicts are unknown" else: atlas_status = "warn" if atlas_issues else "pass" atlas_detail = ( - f"{atlas_summary.get('skill_count', 0)} skills; " - f"{atlas_summary.get('route_collision_count', 0)} route collisions; " - f"{atlas_summary.get('owner_gap_count', 0)} owner gaps; " - f"{atlas_summary.get('stale_count', 0)} stale" + f"{atlas_summary.get('skill_count', 0)} skills, " + f"{atlas_summary.get('actionable_skill_count', atlas_summary.get('skill_count', 0))} actionable; " + f"{actionable_route_collisions} actionable route collisions; " + f"{actionable_owner_gaps} actionable owner gaps; " + f"{actionable_stale} actionable stale; " + f"{atlas_summary.get('non_actionable_issue_count', 0)} scoped non-actionable issues" ) gates.append( gate( @@ -270,6 +429,107 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An ) ) + adoption = data["adoption_drift"] + adoption_summary = adoption.get("summary", {}) + if not adoption: + adoption_status = "warn" + adoption_detail = "adoption drift report is missing; real usage impact is unknown" + elif adoption.get("failures"): + adoption_status = "block" + adoption_detail = f"telemetry privacy or schema failures: {len(adoption.get('failures', []))}" + else: + risk_band = adoption_summary.get("risk_band", "no-data") + adoption_status = "warn" if risk_band in {"no-data", "medium", "high"} else "pass" + adoption_detail = ( + f"{adoption_summary.get('event_count', 0)} metadata events; " + f"adoption {adoption_summary.get('adoption_rate', 0)}; " + f"missed {adoption_summary.get('missed_trigger_count', 0)}; " + f"bad-output {adoption_summary.get('bad_output_count', 0)}; " + f"risk {risk_band}" + ) + gates.append( + gate( + "operations-loop", + "运营回路", + adoption_status, + adoption_detail, + "reports/adoption_drift_report.json", + report_link(output_html, skill_dir, "reports/adoption_drift_report.md"), + ) + ) + + waiver = data["review_waivers"] + waiver_summary = waiver.get("summary", {}) + active_covered = set(waiver_summary.get("covered_gate_keys", []) or []) + prior_blockers = [item for item in gates if item["status"] == "block"] + prior_warnings = [item for item in gates if item["status"] == "warn"] + unwaived_warnings = [item for item in prior_warnings if item["key"] not in active_covered] + if not waiver: + waiver_status = "warn" + waiver_detail = "review waiver ledger is missing; warning acceptance is not auditable" + elif waiver.get("failures"): + waiver_status = "block" + waiver_detail = f"{len(waiver.get('failures', []))} invalid waiver records" + elif prior_blockers: + waiver_status = "block" + waiver_detail = f"{len(prior_blockers)} blocker gates cannot be waived in v0" + elif unwaived_warnings: + waiver_status = "warn" + waiver_detail = ( + f"{waiver_summary.get('active_count', 0)} active waivers; " + f"{len(unwaived_warnings)} warning gates still need reviewer decision" + ) + else: + waiver_status = "pass" + waiver_detail = f"{waiver_summary.get('active_count', 0)} active waivers cover current warnings" + gates.append( + gate( + "review-waivers", + "人工批准", + waiver_status, + waiver_detail, + "reports/review_waivers.json", + report_link(output_html, skill_dir, "reports/review_waivers.md"), + ) + ) + + registry = data["registry"] + install = data["install_simulation"] + if not registry: + if maturity in {"library", "governed"}: + registry_status = "warn" + registry_detail = "registry audit is missing; package metadata is not reviewable" + else: + registry_status = "pass" + registry_detail = "registry audit is optional until team distribution is required" + else: + compatibility = registry.get("package", {}).get("compatibility", {}) + pass_count = sum(1 for status in compatibility.values() if status == "pass") + registry_status = "block" if registry.get("failures") else ("warn" if registry.get("warnings") else "pass") + registry_detail = ( + f"{registry.get('package', {}).get('name', 'package')} " + f"{registry.get('package', {}).get('version', 'n/a')}; " + f"{pass_count}/{len(compatibility)} compatibility entries pass" + ) + if install: + if install.get("failures"): + registry_status = "block" + install_summary = install.get("summary", {}) + registry_detail += ( + f"; install {'pass' if install.get('ok') else 'fail'}" + f" with {install_summary.get('adapter_count', 0)} adapters" + ) + gates.append( + gate( + "registry-audit", + "注册审计", + registry_status, + registry_detail, + "reports/registry_audit.json + reports/install_simulation.json", + report_link(output_html, skill_dir, "reports/registry_audit.md"), + ) + ) + promotion = data["promotion"] migration_path = ROOT / "docs" / "migration-v2.md" if promotion: @@ -280,13 +540,24 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An else: release_status = "warn" release_detail = "promotion decisions are missing; release notes need reviewer confirmation" + upgrade = data["upgrade_check"] + if upgrade: + upgrade_summary = upgrade.get("summary", {}) + if upgrade.get("failures"): + release_status = "block" + elif upgrade.get("warnings") and release_status == "pass": + release_status = "warn" + release_detail += ( + f"; upgrade {upgrade_summary.get('declared_bump', 'n/a')}" + f" declared / {upgrade_summary.get('recommended_bump', 'n/a')} recommended" + ) gates.append( gate( "release-notes", "发布路线", release_status, release_detail, - "reports/promotion_decisions.json + docs/migration-v2.md", + "reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md", report_link(output_html, skill_dir, "reports/promotion_decisions.md") if promotion else str(migration_path), ) ) @@ -301,7 +572,12 @@ def weighted_score(gates: list[dict[str, str]]) -> int: "context-budget": 10, "runtime-matrix": 10, "trust-report": 10, + "permission-gates": 10, + "permission-runtime": 10, "skill-atlas": 10, + "operations-loop": 10, + "review-waivers": 10, + "registry-audit": 10, "release-notes": 10, "intent-canvas": 10, } @@ -322,9 +598,22 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "skill_overview": "reports/skill-overview.html", "review_viewer": "reports/review-viewer.html", "output_eval": "reports/output_quality_scorecard.md", + "output_execution": "reports/output_execution_runs.md", + "output_blind_review": "reports/output_blind_review_pack.md", + "output_review_adjudication": "reports/output_review_adjudication.md", "runtime_conformance": "reports/conformance_matrix.md", "trust_report": "reports/security_trust_report.md", + "permission_policy": "security/permission_policy.md", + "runtime_permissions": "reports/runtime_permission_probes.md", "skill_atlas": "reports/skill_atlas.html", + "compiled_targets": "reports/compiled_targets.md", + "adoption_drift": "reports/adoption_drift_report.md", + "review_waivers": "reports/review_waivers.md", + "review_annotations": "reports/review_annotations.md", + "registry_audit": "reports/registry_audit.md", + "package_verification": "reports/package_verification.md", + "install_simulation": "reports/install_simulation.md", + "upgrade_check": "reports/upgrade_check.md", "migration": "docs/migration-v2.md", "skill_ir": "reports/skill-ir.json", } @@ -339,11 +628,23 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: "intent_dialogue": load_json(reports / "intent-dialogue.json"), "route_scorecard": load_json(reports / "route_scorecard.json"), "output_quality": load_json(reports / "output_quality_scorecard.json"), + "output_execution": load_json(reports / "output_execution_runs.json"), + "output_blind_review": load_json(reports / "output_blind_review_pack.json"), + "output_review_adjudication": load_json(reports / "output_review_adjudication.json"), + "compiled_targets": load_json(reports / "compiled_targets.json"), "conformance": load_json(reports / "conformance_matrix.json"), + "runtime_permissions": load_json(reports / "runtime_permission_probes.json"), "trust": load_json(reports / "security_trust_report.json"), "context_budget": load_json(reports / "context_budget.json"), "promotion": load_json(reports / "promotion_decisions.json"), "atlas": load_json(reports / "skill_atlas.json"), + "adoption_drift": load_json(reports / "adoption_drift_report.json"), + "review_waivers": load_json(reports / "review_waivers.json"), + "review_annotations": load_json(reports / "review_annotations.json"), + "registry": load_json(reports / "registry_audit.json"), + "package_verification": load_json(reports / "package_verification.json"), + "install_simulation": load_json(reports / "install_simulation.json"), + "upgrade_check": load_json(reports / "upgrade_check.json"), "manifest": load_json(skill_dir / "manifest.json"), "frontmatter": parse_frontmatter(skill_dir / "SKILL.md"), "interface": load_yaml(skill_dir / "agents" / "interface.yaml"), @@ -353,25 +654,66 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: overview = data["overview"] output = data["output_quality"].get("summary", {}) + output_execution = data["output_execution"].get("summary", {}) + output_blind = data["output_blind_review"].get("summary", {}) + output_review = data["output_review_adjudication"].get("summary", {}) + compiled = data["compiled_targets"].get("summary", {}) conformance = data["conformance"].get("summary", {}) + runtime_permissions = data["runtime_permissions"].get("summary", {}) trust = data["trust"].get("summary", {}) atlas = data["atlas"].get("summary", {}) + adoption = data["adoption_drift"].get("summary", {}) + waivers = data["review_waivers"].get("summary", {}) + annotations = data["review_annotations"].get("summary", {}) + registry = data["registry"].get("package", {}) + package_verification = data["package_verification"].get("summary", {}) + install_simulation = data["install_simulation"].get("summary", {}) + upgrade = data["upgrade_check"].get("summary", {}) cards = [ { "label": "Skill IR", "value": str(overview.get("skill_ir", {}).get("schema_version", "missing")), "detail": f"{overview.get('skill_ir', {}).get('target_count', 0)} targets in platform-neutral contract", }, + { + "label": "Compiler", + "value": f"{compiled.get('pass_count', 0)}/{compiled.get('target_count', 0)}", + "detail": "target contracts compiled from Skill IR", + }, { "label": "Output Delta", "value": str(output.get("delta", "n/a")), "detail": f"{output.get('case_count', 0)} cases; {output.get('file_backed_case_count', 0)} file-backed", }, + { + "label": "Exec Runs", + "value": str(output_execution.get("variant_run_count", 0)), + "detail": ( + f"command {output_execution.get('command_executed_count', 0)}; " + f"model {output_execution.get('model_executed_count', 0)}; " + f"recorded {output_execution.get('recorded_fixture_count', 0)}" + ), + }, + { + "label": "Blind A/B", + "value": str(output_blind.get("pair_count", 0)), + "detail": "review pairs hide baseline vs with-skill labels", + }, + { + "label": "Review A/B", + "value": f"{output_review.get('judgment_count', 0)}/{output_review.get('pair_count', 0)}", + "detail": f"adjudication decisions; pending {output_review.get('pending_count', 0)}", + }, { "label": "Runtime", "value": f"{conformance.get('pass_count', 0)}/{conformance.get('target_count', 0)}", "detail": "target conformance pass rate", }, + { + "label": "Perm Probe", + "value": f"{runtime_permissions.get('metadata_fallback_count', 0)}/{runtime_permissions.get('target_count', 0)}", + "detail": f"{runtime_permissions.get('native_enforcement_count', 0)} native enforcement targets", + }, { "label": "Trust", "value": str(trust.get("secret_findings", 0)), @@ -382,6 +724,41 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: "value": str(atlas.get("route_collision_count", 0)), "detail": f"{atlas.get('skill_count', 0)} scanned skills; route collisions", }, + { + "label": "Drift", + "value": str(adoption.get("risk_band", "n/a")), + "detail": f"{adoption.get('event_count', 0)} metadata events; {adoption.get('missed_trigger_count', 0)} missed triggers", + }, + { + "label": "Waivers", + "value": str(waivers.get("active_count", 0)), + "detail": f"{waivers.get('covered_gate_count', 0)} gates covered; human risk decisions", + }, + { + "label": "Notes", + "value": f"{annotations.get('open_count', 0)}/{annotations.get('annotation_count', 0)}", + "detail": f"{annotations.get('open_blocker_count', 0)} open blocker annotations", + }, + { + "label": "Registry", + "value": str(registry.get("version", "n/a")), + "detail": f"{len(registry.get('targets', []))} targets; {registry.get('license', 'no license')} license", + }, + { + "label": "Archive", + "value": "pass" if data["package_verification"].get("ok") else "n/a", + "detail": f"{package_verification.get('archive_entry_count', 0)} zip entries; package verification", + }, + { + "label": "Install", + "value": "pass" if data["install_simulation"].get("ok") else "n/a", + "detail": f"{install_simulation.get('adapter_count', 0)} adapters readable; local install simulation", + }, + { + "label": "Upgrade", + "value": str(upgrade.get("recommended_bump", "n/a")), + "detail": f"declared {upgrade.get('declared_bump', 'n/a')}; {upgrade.get('breaking_change_count', 0)} breaking changes", + }, ] return cards @@ -430,9 +807,267 @@ def render_issue_list(title: str, items: list[dict[str, str]]) -> str: return f"

    {html.escape(title)}

      {body}
    " +ACTION_GUIDANCE: dict[str, dict[str, str]] = { + "intent-canvas": { + "summary": "收紧真实任务、输入、输出、排除项和成功标准。", + "why": "低 intent confidence 会让后续 Skill IR、输出评测和 Review Studio 结论建立在模糊意图上。", + "source_fix": "reports/intent-dialogue.md + reports/intent-confidence.md", + "source_paths": [ + {"path": "reports/intent-dialogue.md", "label": "intent dialogue", "kind": "report", "patterns": ["# Intent"]}, + {"path": "reports/intent-confidence.md", "label": "intent confidence", "kind": "report", "patterns": ["# Intent"]}, + ], + "verification": "python3 scripts/yao.py intent-confidence .", + }, + "trigger-lab": { + "summary": "修正 route scorecard 中的误触发、漏触发或 ambiguous case。", + "why": "触发错误会让正确 Skill 失活，或让相邻 Skill 被错误调用。", + "source_fix": "SKILL.md frontmatter description + evals/*/trigger_cases.json", + "source_paths": [ + {"path": "SKILL.md", "label": "frontmatter description", "kind": "source", "patterns": ["description:"]}, + {"path": "evals/trigger_cases.json", "label": "trigger eval cases", "kind": "eval", "patterns": ["should_trigger"]}, + {"path": "reports/route_scorecard.md", "label": "route scorecard", "kind": "report", "patterns": ["# Route"]}, + ], + "verification": "python3 scripts/build_confusion_matrix.py", + }, + "output-lab": { + "summary": "补足 output eval 的 case 数、file-backed case、near-neighbor case 和 boundary case。", + "why": "没有输出质量证据时，Skill 只能证明会触发，不能证明输出真的更好。", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md", + "source_paths": [ + {"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ["case_id"]}, + {"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]}, + {"path": "reports/output_execution_runs.md", "label": "output execution runs", "kind": "report", "patterns": ["# Output Execution"]}, + {"path": "reports/output_blind_review_pack.md", "label": "blind A/B review pack", "kind": "report", "patterns": ["# Output Blind"]}, + {"path": "reports/output_review_adjudication.md", "label": "review adjudication", "kind": "report", "patterns": ["# Output Review"]}, + ], + "verification": "python3 scripts/run_output_execution.py", + }, + "context-budget": { + "summary": "压缩入口与高成本 references，保留最小可路由上下文。", + "why": "上下文成本过高会降低加载稳定性，并挤压用户任务材料的预算。", + "source_fix": "SKILL.md + references/", + "source_paths": [ + {"path": "SKILL.md", "label": "entrypoint", "kind": "source", "patterns": ["# Yao Meta Skill"]}, + {"path": "reports/context_budget.md", "label": "context budget", "kind": "report", "patterns": ["# Context"]}, + ], + "verification": "python3 scripts/render_context_reports.py", + }, + "runtime-matrix": { + "summary": "修复目标端结构、metadata、相对路径、fallback 或 adapter target 声明。", + "why": "runtime conformance 失败意味着包可能被目标客户端错误加载或静默降级。", + "source_fix": "agents/interface.yaml + reports/conformance_matrix.md", + "source_paths": [ + {"path": "agents/interface.yaml", "label": "portable interface", "kind": "source", "patterns": ["adapter_targets"]}, + {"path": "reports/conformance_matrix.md", "label": "conformance matrix", "kind": "report", "patterns": ["# Runtime"]}, + ], + "verification": "python3 scripts/run_conformance_suite.py .", + }, + "trust-report": { + "summary": "处理脚本 help surface、依赖 pin、network policy、secret 和权限声明。", + "why": "团队分发时，脚本和依赖是主要供应链风险面，warning 必须有明确处置。", + "source_fix": "reports/security_trust_report.md + security/*.md + scripts/", + "source_paths": [ + {"path": "reports/security_trust_report.md", "label": "trust report", "kind": "report", "patterns": ["# Security"]}, + {"path": "security/script_policy.md", "label": "script policy", "kind": "policy", "patterns": ["# Script"]}, + {"path": "security/network_policy.md", "label": "network policy", "kind": "policy", "patterns": ["# Network"]}, + ], + "verification": "python3 scripts/trust_check.py .", + }, + "permission-gates": { + "summary": "补齐高权限能力的 reviewer、scope、reason、expires_at 和目标端 enforcement 说明。", + "why": "权限契约只有在批准人、有效期和目标端处置方式明确时，才能支撑 governed release。", + "source_fix": "security/permission_policy.json + security/permission_policy.md", + "source_paths": [ + {"path": "security/permission_policy.json", "label": "permission approvals", "kind": "policy", "patterns": ["approved"]}, + {"path": "security/permission_policy.md", "label": "permission method", "kind": "policy", "patterns": ["# Permission"]}, + ], + "verification": "python3 scripts/trust_check.py .", + }, + "permission-runtime": { + "summary": "生成并修复目标包的 runtime permission probe 报告。", + "why": "目标端即使只能提供 metadata fallback，也必须明确 native enforcement 缺口、表示位置和 operator note。", + "source_fix": "dist/targets/*/adapter.json + reports/runtime_permission_probes.md", + "source_paths": [ + {"path": "reports/runtime_permission_probes.md", "label": "runtime permission probes", "kind": "report", "patterns": ["# Runtime"]}, + {"path": "dist/targets/openai/adapter.json", "label": "OpenAI adapter", "kind": "package", "patterns": ["target_permission_contract"]}, + {"path": "dist/targets/claude/adapter.json", "label": "Claude adapter", "kind": "package", "patterns": ["target_permission_contract"]}, + {"path": "dist/targets/generic/adapter.json", "label": "generic adapter", "kind": "package", "patterns": ["target_permission_contract"]}, + ], + "verification": "python3 scripts/probe_runtime_permissions.py . --package-dir dist", + }, + "skill-atlas": { + "summary": "处理 portfolio 里的路由冲突、owner 缺口、stale skill 和重复能力。", + "why": "单个 Skill 质量很高仍可能在团队 skill library 中被相邻 Skill 冲突削弱。", + "source_fix": "reports/skill_atlas.html + skill_atlas/catalog.json", + "source_paths": [ + {"path": "skill_atlas/catalog.json", "label": "skill atlas catalog", "kind": "atlas", "patterns": ["summary"]}, + {"path": "skill_atlas/policy.json", "label": "atlas scope policy", "kind": "policy", "patterns": ["scope"]}, + {"path": "reports/skill_atlas.html", "label": "skill atlas report", "kind": "report", "patterns": ["Skill Atlas"]}, + ], + "verification": "python3 scripts/build_skill_atlas.py --workspace-root .", + }, + "operations-loop": { + "summary": "记录 metadata-only 使用事件，或明确当前 release 缺少真实使用信号。", + "why": "没有运营回路时，reviewer 无法判断采用率、误触发、坏输出和 review overdue 的真实影响。", + "source_fix": "reports/adoption_drift_report.md", + "source_paths": [ + {"path": "reports/adoption_drift_report.md", "label": "adoption drift report", "kind": "report", "patterns": ["# Adoption"]}, + {"path": "references/telemetry-drift-method.md", "label": "telemetry method", "kind": "method", "patterns": ["# Telemetry"]}, + ], + "verification": "python3 scripts/render_adoption_drift_report.py . --record-event skill_activation --activation-type explicit --outcome accepted", + }, + "review-waivers": { + "summary": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "why": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "source_fix": "reports/review_waivers.md", + "source_paths": [ + {"path": "reports/review_waivers.md", "label": "waiver ledger", "kind": "report", "patterns": ["# Review"]}, + {"path": "references/review-waiver-method.md", "label": "waiver method", "kind": "method", "patterns": ["# Review"]}, + ], + "verification": "python3 scripts/render_review_waivers.py .", + }, + "registry-audit": { + "summary": "补齐 registry package metadata、checksum、license、owner、review cadence 和 install evidence。", + "why": "分发元数据不完整时，团队无法安全安装、升级或追溯包体来源。", + "source_fix": "registry/ + reports/registry_audit.md", + "source_paths": [ + {"path": "registry/packages/yao-meta-skill.json", "label": "registry package", "kind": "registry", "patterns": ["version"]}, + {"path": "reports/registry_audit.md", "label": "registry audit", "kind": "report", "patterns": ["# Registry"]}, + {"path": "reports/install_simulation.md", "label": "install simulation", "kind": "report", "patterns": ["# Install"]}, + ], + "verification": "python3 scripts/registry_audit.py .", + }, + "release-notes": { + "summary": "确认 promotion、upgrade diff、breaking changes、migration guide 和 known limitations。", + "why": "发布说明不完整会让使用者无法判断升级风险和迁移动作。", + "source_fix": "reports/upgrade_check.md + docs/migration-v2.md", + "source_paths": [ + {"path": "reports/upgrade_check.md", "label": "upgrade check", "kind": "report", "patterns": ["# Upgrade"]}, + {"path": "docs/migration-v2.md", "label": "migration guide", "kind": "docs", "patterns": ["# Migration"]}, + {"path": "reports/promotion_decisions.md", "label": "promotion decisions", "kind": "report", "patterns": ["# Promotion"]}, + ], + "verification": "python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json", + }, +} + + +def build_review_actions(gates: list[dict[str, str]], skill_dir: Path, output_html: Path) -> list[dict[str, Any]]: + actions: list[dict[str, Any]] = [] + for gate_item in gates: + if gate_item["status"] == "pass": + continue + guidance = ACTION_GUIDANCE.get( + gate_item["key"], + { + "summary": "打开证据报告，修复当前 gate 暴露的问题。", + "why": "该 gate 未通过，release reviewer 需要明确处置动作。", + "source_fix": gate_item.get("evidence", ""), + "source_paths": [], + "verification": "python3 scripts/render_review_studio.py .", + }, + ) + refs = source_refs(skill_dir, output_html, guidance.get("source_paths", [])) + actions.append( + { + "gate_key": gate_item["key"], + "label": gate_item["label"], + "status": gate_item["status"], + "priority": "blocker" if gate_item["status"] == "block" else "warning", + "summary": guidance["summary"], + "why": guidance["why"], + "source_fix": guidance["source_fix"], + "source_refs": refs, + "evidence": gate_item.get("evidence", ""), + "evidence_link": gate_item.get("link", ""), + "verification_command": guidance["verification"], + } + ) + return actions + + +def render_action_source_refs(refs: list[dict[str, Any]]) -> str: + if not refs: + return "

    暂无结构化 source refs；请先打开证据报告。

    " + items = [] + for ref in refs: + line_suffix = f":{ref['line']}" if ref.get("line") else "" + label = f"{ref['path']}{line_suffix}" + if ref.get("exists") and ref.get("link"): + path_html = f"{html.escape(label)}" + else: + path_html = f"{html.escape(label)} · missing" + items.append( + "
  • " + f"{path_html}" + f"{html.escape(ref.get('label', 'source'))} · {html.escape(ref.get('kind', 'source'))}" + "
    • " + ) + return "" + + +def render_review_actions(actions: list[dict[str, Any]]) -> str: + if not actions: + return "

    当前没有 blocker 或 warning。保持现有证据链即可。

    " + cards = [] + for item in actions: + link_html = f"打开证据" if item.get("evidence_link") else "" + source_refs_html = render_action_source_refs(item.get("source_refs", [])) + cards.append( + "
    " + f"
    {html.escape(status_label(item['status']))}

    {html.escape(item['label'])}

    " + f"

    {html.escape(item['summary'])}

    " + f"{html.escape(item['why'])}" + f"
    修复位置
    {html.escape(item['source_fix'])}
    " + f"
    验证命令
    {html.escape(item['verification_command'])}
    " + f"{source_refs_html}" + f"
    {html.escape(item['evidence'])} {link_html}
    " + "
    " + ) + return "".join(cards) + + +def render_review_annotations_panel(annotations_report: dict[str, Any]) -> str: + annotations = annotations_report.get("annotations", []) if isinstance(annotations_report, dict) else [] + if not annotations: + return "

    当前没有 reviewer 批注。

    " + cards = [] + for item in annotations: + line_suffix = f":{item['line']}" if item.get("line") else "" + target_label = f"{item.get('target_path', '')}{line_suffix}" + meta = " · ".join( + part + for part in [ + str(item.get("gate_key", "")), + str(item.get("reviewer", "")), + str(item.get("created_at", "")), + ] + if part + ) + cards.append( + "
    " + f"
    {html.escape(str(item.get('severity', 'note')))} · {html.escape(str(item.get('status', 'open')))}" + f"

    {html.escape(str(item.get('id', 'annotation')))}

    " + f"

    {html.escape(str(item.get('body', '')))}

    " + f"
    位置
    {html.escape(target_label)}
    " + f"
    Gate
    {html.escape(str(item.get('gate_key', '')))}
    " + f"
    建议
    {html.escape(str(item.get('suggested_action', '') or '无'))}
    " + f"{html.escape(meta)}" + f"
    {html.escape(str(item.get('source_excerpt', '')))}
    " + "
    " + ) + return "".join(cards) + + def render_html(report: dict[str, Any]) -> str: summary = report["summary"] gates = report["gates"] + gate_details = {item["key"]: item["detail"] for item in gates} blockers = report["blockers"] warnings = report["warnings"] insights = insight_cards(report["data"]) @@ -446,9 +1081,16 @@ def render_html(report: dict[str, Any]) -> str: ("#intent", "意图画布"), ("#trigger", "触发实验"), ("#output", "输出实验"), + ("#actions", "修复动作"), + ("#annotations", "审查批注"), ("#runtime", "运行矩阵"), ("#trust", "信任报告"), + ("#permissions", "权限批准"), + ("#permission-probes", "权限探针"), ("#atlas", "组合治理"), + ("#telemetry", "运营回路"), + ("#waivers", "人工批准"), + ("#registry", "注册审计"), ("#release", "发布路线"), ] nav_html = "".join(f"{label}" for href, label in nav) @@ -456,10 +1098,27 @@ def render_html(report: dict[str, Any]) -> str: metrics_html = render_insights(insights) blockers_html = render_issue_list("阻断事项", blockers) warnings_html = render_issue_list("关注事项", warnings) + actions_html = render_review_actions(report["review_actions"]) + annotations_html = render_review_annotations_panel(report["data"].get("review_annotations", {})) output_summary = report["data"]["output_quality"].get("summary", {}) + output_execution_summary = report["data"]["output_execution"].get("summary", {}) + output_blind_summary = report["data"]["output_blind_review"].get("summary", {}) + output_review_summary = report["data"]["output_review_adjudication"].get("summary", {}) conformance_summary = report["data"]["conformance"].get("summary", {}) + compiled_summary = report["data"]["compiled_targets"].get("summary", {}) trust_summary = report["data"]["trust"].get("summary", {}) + runtime_permissions_summary = report["data"]["runtime_permissions"].get("summary", {}) atlas_summary = report["data"]["atlas"].get("summary", {}) + adoption_summary = report["data"]["adoption_drift"].get("summary", {}) + waiver_summary = report["data"]["review_waivers"].get("summary", {}) + annotation_summary = report["data"]["review_annotations"].get("summary", {}) + annotation_caption = ( + f"{annotation_summary.get('annotation_count', 0)} 条批注；" + f"开放 {annotation_summary.get('open_count', 0)}；" + f"阻断 {annotation_summary.get('open_blocker_count', 0)}" + ) + registry_package = report["data"]["registry"].get("package", {}) + package_summary = report["data"]["package_verification"].get("summary", {}) evidence_html = "".join( f"
  • {html.escape(key)}{html.escape(value)}
    • " for key, value in report["evidence_paths"].items() @@ -582,8 +1241,89 @@ def render_html(report: dict[str, Any]) -> str: gap: 3px; }} .muted {{ color: var(--muted); }} + .actions-grid {{ + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + }} + .annotations-grid {{ + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: 16px; + }} + .action-card {{ + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 10px; + min-width: 0; + }} + .action-card.warn {{ border-left: 4px solid var(--warn); }} + .action-card.block {{ border-left: 4px solid var(--block); }} + .annotation-card {{ + border: 1px solid var(--line); + border-radius: 8px; + padding: 18px; + background: #fff; + display: grid; + gap: 10px; + min-width: 0; + }} + .annotation-card.warning {{ border-left: 4px solid var(--warn); }} + .annotation-card.blocker {{ border-left: 4px solid var(--block); }} + .annotation-card.resolved {{ opacity: .72; }} + .action-card span, + .annotation-card span, + .action-card small, + .annotation-card small, + .action-card footer, + .annotation-card footer, + .action-card dd {{ + color: var(--muted); + font-size: 14px; + overflow-wrap: anywhere; + }} + .annotation-card dd {{ color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }} + .action-card dl, .annotation-card dl {{ + display: grid; + grid-template-columns: 80px minmax(0, 1fr); + gap: 6px 10px; + margin: 0; + }} + .action-card dt, .annotation-card dt {{ color: var(--ink); font-size: 14px; }} + .action-card dd, .annotation-card dd {{ margin: 0; }} + .source-ref-list {{ + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 6px; + }} + .source-ref-list li {{ + display: grid; + gap: 2px; + min-width: 0; + padding: 8px 0 0; + border-top: 1px solid var(--line); + }} + .source-ref-list a, + .source-ref-list span {{ + overflow-wrap: anywhere; + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + }} + .source-ref-list small {{ + font-size: 12px; + color: var(--muted); + }} + code {{ + font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; + font-size: 13px; + }} @media (max-width: 980px) {{ - .metrics, .gates, .twocol {{ grid-template-columns: 1fr; }} + .metrics, .gates, .twocol, .actions-grid, .annotations-grid {{ grid-template-columns: 1fr; }} main {{ padding: 32px 18px 60px; }} nav {{ justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; }} nav a {{ flex: 0 0 auto; }} @@ -619,6 +1359,17 @@ def render_html(report: dict[str, Any]) -> str: {warnings_html}
    +
    +

    修复动作

    +
    {actions_html}
    +
    + +
    +

    审查批注

    +

    当前批注：{html.escape(annotation_caption)}

    +
    {annotations_html}
    +
    +

    意图画布

    @@ -637,12 +1388,32 @@ def render_html(report: dict[str, Any]) -> str:

    输出实验

    {html.escape(str(output_summary))}

    -

    发布标准

    Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor 和 boundary case。

    +

    执行证据

    {html.escape(str(output_execution_summary or 'output execution report missing'))}

    +
    + +
    +

    盲评包

    {html.escape(str(output_blind_summary or 'blind A/B review pack missing'))}

    +

    审定报告

    {html.escape(str(output_review_summary or 'review adjudication report missing'))}

    +
    + +
    +

    发布标准

    Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor、boundary case、execution evidence 和 blind A/B review pack。

    +

    人工结论

    没有 reviewer 决策时只显示 pending；只有真实决策文件会进入一致率和分歧统计。

    +
    + +
    +

    评审方式

    先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

    +

    运行方式

    reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。

    运行矩阵

    {html.escape(str(conformance_summary))}

    -

    上下文

    {html.escape(gates[3]['detail'])}

    +

    目标编译

    {html.escape(str(compiled_summary or 'compiled target report missing'))}

    +
    + +
    +

    上下文

    {html.escape(gate_details.get('context-budget', 'context budget missing'))}

    +

    编译证据

    Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

    @@ -650,14 +1421,39 @@ def render_html(report: dict[str, Any]) -> str:

    安全边界

    高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

    +
    +

    权限批准

    {html.escape(gate_details.get('permission-gates', 'permission governance missing'))}

    +

    批准策略

    高权限能力需要 reviewer、scope、reason、expires_at 和 openai/claude/generic 目标端 enforcement 说明。

    +
    + +
    +

    权限探针

    {html.escape(gate_details.get('permission-runtime', 'runtime permission probes missing'))}

    +

    运行边界

    {html.escape(str(runtime_permissions_summary or 'runtime permission probe summary missing'))}

    +
    +
    -

    组合治理

    {html.escape(gates[6]['detail'])}

    -

    下一动作

    优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再做 registry 或 telemetry。

    +

    组合治理

    {html.escape(gate_details.get('skill-atlas', 'skill atlas missing'))}

    +

    下一动作

    优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再用运营回路判断真实影响。

    +
    + +
    +

    运营回路

    {html.escape(gate_details.get('operations-loop', 'adoption drift report missing'))}

    +

    漂移信号

    {html.escape(str(adoption_summary or 'no adoption drift summary'))}

    +
    + +
    +

    人工批准

    {html.escape(gate_details.get('review-waivers', 'review waiver ledger missing'))}

    +

    批准台账

    {html.escape(str(waiver_summary or 'no review waiver summary'))}

    +
    + +
    +

    注册审计

    {html.escape(gate_details.get('registry-audit', 'registry audit missing'))}

    +

    包体元数据

    {html.escape(str(registry_package))}

    -

    发布路线

    {html.escape(gates[7]['detail'])}

    -

    世界级缺口

    下一阶段应继续推进 IR-first compiler、registry audit、telemetry drift loop，以及更严格的 governed trust gates。

    +

    发布路线

    {html.escape(gate_details.get('release-notes', 'release notes missing'))}

    +

    包体验证

    {html.escape(str(package_summary or 'package verification missing'))}

    @@ -674,8 +1470,12 @@ def render_review_studio(skill_dir: Path, output_html: Path | None = None, outpu data = load_review_data(skill_dir) gates = build_gates(skill_dir, output_html, data) blockers, warnings = add_blockers_from_gate(gates) + review_actions = build_review_actions(gates, skill_dir, output_html) score = weighted_score(gates) - decision = "blocked" if blockers else ("review" if warnings else "ready") + annotation_summary = data["review_annotations"].get("summary", {}) + open_annotation_blockers = int(annotation_summary.get("open_blocker_count", 0) or 0) + open_annotation_warnings = int(annotation_summary.get("open_warning_count", 0) or 0) + decision = "blocked" if blockers or open_annotation_blockers else ("review" if warnings or open_annotation_warnings else "ready") report = { "schema_version": "2.0", "ok": True, @@ -686,10 +1486,16 @@ def render_review_studio(skill_dir: Path, output_html: Path | None = None, outpu "gate_count": len(gates), "blocker_count": len(blockers), "warning_count": len(warnings), + "action_count": len(review_actions), + "annotation_count": int(annotation_summary.get("annotation_count", 0) or 0), + "open_annotation_count": int(annotation_summary.get("open_count", 0) or 0), + "open_annotation_blocker_count": open_annotation_blockers, + "open_annotation_warning_count": open_annotation_warnings, }, "gates": gates, "blockers": blockers, "warnings": warnings, + "review_actions": review_actions, "evidence_paths": evidence_paths(skill_dir), "data": data, "artifacts": { diff --git a/scripts/render_review_waivers.py b/scripts/render_review_waivers.py new file mode 100644 index 0000000..70eb23d --- /dev/null +++ b/scripts/render_review_waivers.py @@ -0,0 +1,281 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +KNOWN_GATE_KEYS = { + "intent-canvas", + "trigger-lab", + "output-lab", + "context-budget", + "runtime-matrix", + "trust-report", + "permission-gates", + "permission-runtime", + "skill-atlas", + "operations-loop", + "registry-audit", + "release-notes", +} +VALID_DECISIONS = {"accepted-risk", "false-positive", "temporary-exception"} +MIN_REASON_CHARS = 20 + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def today_from(value: str | None) -> date: + if not value: + return date.today() + return date.fromisoformat(value[:10]) + + +def parse_date(value: str) -> date | None: + try: + return date.fromisoformat(value[:10]) + except (TypeError, ValueError): + return None + + +def waiver_id(entry: dict[str, Any]) -> str: + raw = "|".join( + [ + str(entry.get("gate_key", "")), + str(entry.get("reviewer", "")), + str(entry.get("created_at", "")), + str(entry.get("reason", "")), + ] + ) + return hashlib.sha1(raw.encode("utf-8")).hexdigest()[:12] + + +def normalize_waiver(entry: dict[str, Any]) -> dict[str, Any]: + normalized = { + "id": str(entry.get("id") or ""), + "gate_key": str(entry.get("gate_key") or ""), + "decision": str(entry.get("decision") or "accepted-risk"), + "reviewer": str(entry.get("reviewer") or ""), + "reason": str(entry.get("reason") or ""), + "created_at": str(entry.get("created_at") or ""), + "expires_at": str(entry.get("expires_at") or ""), + "evidence": str(entry.get("evidence") or ""), + "scope": str(entry.get("scope") or "current-release"), + } + normalized["id"] = normalized["id"] or waiver_id(normalized) + return normalized + + +def add_waiver(existing: list[dict[str, Any]], args: argparse.Namespace, today: date) -> list[dict[str, Any]]: + created_at = args.created_at or today.isoformat() + entry = normalize_waiver( + { + "gate_key": args.gate_key, + "decision": args.decision, + "reviewer": args.reviewer, + "reason": args.reason, + "created_at": created_at, + "expires_at": args.expires_at, + "evidence": args.evidence or "", + "scope": args.scope, + } + ) + return [*existing, entry] + + +def validate_waivers(waivers: list[dict[str, Any]], today: date) -> tuple[list[dict[str, Any]], list[str], list[str]]: + normalized = [] + failures = [] + warnings = [] + seen_ids = set() + + for index, raw in enumerate(waivers, start=1): + entry = normalize_waiver(raw) + entry_failures = [] + entry_warnings = [] + if entry["id"] in seen_ids: + entry_failures.append("duplicate waiver id") + seen_ids.add(entry["id"]) + if entry["gate_key"] not in KNOWN_GATE_KEYS: + entry_failures.append(f"unknown gate_key: {entry['gate_key'] or ''}") + if entry["decision"] not in VALID_DECISIONS: + entry_failures.append(f"invalid decision: {entry['decision'] or ''}") + if not entry["reviewer"]: + entry_failures.append("reviewer is required") + if len(entry["reason"]) < MIN_REASON_CHARS: + entry_failures.append(f"reason must be at least {MIN_REASON_CHARS} characters") + created_at = parse_date(entry["created_at"]) + if created_at is None: + entry_failures.append("created_at must be ISO date") + expires_at = parse_date(entry["expires_at"]) + if expires_at is None: + entry_failures.append("expires_at must be ISO date") + elif expires_at < today: + entry_warnings.append("waiver is expired") + entry["status"] = "invalid" if entry_failures else ("expired" if entry_warnings else "active") + entry["validation"] = {"failures": entry_failures, "warnings": entry_warnings} + normalized.append(entry) + failures.extend([f"waiver {index} ({entry['id']}): {item}" for item in entry_failures]) + warnings.extend([f"waiver {index} ({entry['id']}): {item}" for item in entry_warnings]) + + return normalized, failures, warnings + + +def render_report( + skill_dir: Path, + waivers_json: Path | None = None, + output_json: Path | None = None, + output_md: Path | None = None, + generated_at: str | None = None, + add_args: argparse.Namespace | None = None, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports = skill_dir / "reports" + reports.mkdir(parents=True, exist_ok=True) + output_json = output_json or reports / "review_waivers.json" + output_md = output_md or reports / "review_waivers.md" + source_json = waivers_json or output_json + today = today_from(generated_at) + payload = load_json(source_json) + raw_waivers = payload.get("waivers", []) if isinstance(payload.get("waivers", []), list) else [] + if add_args is not None: + raw_waivers = add_waiver(raw_waivers, add_args, today) + waivers, failures, warnings = validate_waivers(raw_waivers, today) + active = [item for item in waivers if item["status"] == "active"] + expired = [item for item in waivers if item["status"] == "expired"] + invalid = [item for item in waivers if item["status"] == "invalid"] + covered_gate_keys = sorted({item["gate_key"] for item in active}) + report = { + "schema_version": "1.0", + "ok": not failures, + "skill_dir": display_path(skill_dir), + "generated_at": generated_at or today.isoformat(), + "summary": { + "waiver_count": len(waivers), + "active_count": len(active), + "expired_count": len(expired), + "invalid_count": len(invalid), + "covered_gate_count": len(covered_gate_keys), + "covered_gate_keys": covered_gate_keys, + }, + "policy": { + "blocker_waivers_allowed": False, + "minimum_reason_chars": MIN_REASON_CHARS, + "expires_required": True, + "known_gate_keys": sorted(KNOWN_GATE_KEYS), + }, + "waivers": waivers, + "failures": failures, + "warnings": warnings, + "artifacts": { + "json": display_path(output_json), + "markdown": display_path(output_md), + }, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + return report + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Review Waivers", + "", + f"- OK: `{report['ok']}`", + f"- Waivers: `{summary['waiver_count']}`", + f"- Active: `{summary['active_count']}`", + f"- Expired: `{summary['expired_count']}`", + f"- Invalid: `{summary['invalid_count']}`", + f"- Covered gates: `{', '.join(summary['covered_gate_keys']) or 'none'}`", + "", + "## Policy", + "", + "- Blocker waivers allowed: `False`", + f"- Minimum reason chars: `{report['policy']['minimum_reason_chars']}`", + "- Expiry is required for every waiver.", + "", + "## Waivers", + "", + ] + if not report["waivers"]: + lines.append("- None") + else: + lines.extend(["| ID | Gate | Decision | Reviewer | Status | Expires | Reason |", "| --- | --- | --- | --- | --- | --- | --- |"]) + for item in report["waivers"]: + reason = str(item["reason"]).replace("|", "\\|") + lines.append( + f"| `{item['id']}` | `{item['gate_key']}` | `{item['decision']}` | {item['reviewer']} | `{item['status']}` | `{item['expires_at']}` | {reason} |" + ) + lines.extend(["", "## Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) + lines.extend(["", "## Warnings", ""]) + lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"]) + return "\n".join(lines).strip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render or update Review Studio waiver evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--waivers-json") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + parser.add_argument("--generated-at") + parser.add_argument("--add-waiver", action="store_true") + parser.add_argument("--gate-key", choices=sorted(KNOWN_GATE_KEYS)) + parser.add_argument("--decision", choices=sorted(VALID_DECISIONS), default="accepted-risk") + parser.add_argument("--reviewer") + parser.add_argument("--reason") + parser.add_argument("--expires-at") + parser.add_argument("--created-at") + parser.add_argument("--evidence") + parser.add_argument("--scope", default="current-release") + args = parser.parse_args() + + add_args = None + if args.add_waiver: + required = { + "--gate-key": args.gate_key, + "--reviewer": args.reviewer, + "--reason": args.reason, + "--expires-at": args.expires_at, + } + missing = [name for name, value in required.items() if not value] + if missing: + print(json.dumps({"ok": False, "failures": [f"Missing required fields for --add-waiver: {', '.join(missing)}"]}, ensure_ascii=False, indent=2)) + raise SystemExit(2) + add_args = args + + payload = render_report( + Path(args.skill_dir), + waivers_json=Path(args.waivers_json).resolve() if args.waivers_json else None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + output_md=Path(args.output_md).resolve() if args.output_md else None, + generated_at=args.generated_at, + add_args=add_args, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_skill_overview.py b/scripts/render_skill_overview.py index ff2a404..b0f5341 100644 --- a/scripts/render_skill_overview.py +++ b/scripts/render_skill_overview.py @@ -69,6 +69,14 @@ TEXT_EN = { "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。": "Use this report to quickly understand the generated skill's role, principles, trigger boundary, and deliverables.", "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。": "Clarify the recurring job, real input shape, and deliverable output before adding references, scripts, or evals.", "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。": "If the request is still fuzzy, tighten the boundary through intent dialogue before expanding the package.", + "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。": "Output Review Adjudication is generated to record blind-review decisions, agreement rate, and pending cases.", + "已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。": "Output Execution Runs is generated to distinguish recorded fixtures, command runs, and model-run evidence.", + "尚未生成盲评审定报告。": "The blind review adjudication report has not been generated yet.", + "尚未生成输出执行证据报告。": "The output execution evidence report has not been generated yet.", + "先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。": "Record the reviewer's A/B choice before opening the answer key and calculating agreement.", + "缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。": "When real reviewer decisions are missing, show pending status instead of fabricating human conclusions.", + "recorded fixture 只能证明可复现样本，不等同于模型执行。": "A recorded fixture proves reproducible samples only; it is not model execution.", + "只有 provider runner 返回 model metadata 时才计入 model-executed。": "Only provider runners that return model metadata count as model-executed.", } MODE_ZH = { @@ -289,6 +297,10 @@ def render_html(summary: dict) -> str: risk = summary.get("risk_governance", {}) assets = summary.get("package_assets", {}) roadmap = summary.get("iteration_roadmap", {}) + output_execution = summary.get("output_execution", {}) + output_execution_summary = output_execution.get("summary", {}) + output_review = summary.get("output_review_adjudication", {}) + output_review_summary = output_review.get("summary", {}) hero_meta = [ (f"技能名称：{summary['name']}", f"Skill name: {summary['name']}"), (f"成熟度：{mode_zh(metadata.get('maturity_tier', 'scaffold'))}", f"Maturity: {metadata.get('maturity_tier', 'scaffold')}"), @@ -312,6 +324,25 @@ def render_html(summary: dict) -> str: [{"name": "强项", "signal": item, "response": "保留并复用"} for item in quality.get("strengths", [])[:3]] + [{"name": "缺口", "signal": item, "response": "纳入下一轮修复"} for item in quality.get("gaps", [])[:3]] ) + if output_review_summary: + agreement = output_review_summary.get("agreement_rate") + review_items = [ + f"评审进度：{output_review_summary.get('judgment_count', 0)} / {output_review_summary.get('pair_count', 0)}", + f"待评审：{output_review_summary.get('pending_count', 0)}", + f"一致率：{agreement if agreement is not None else '暂无'}", + f"非法决策：{output_review_summary.get('invalid_decision_count', 0)}", + ] + else: + review_items = ["尚未生成盲评审定报告。"] + if output_execution_summary: + execution_items = [ + f"变体运行：{output_execution_summary.get('variant_run_count', 0)}", + f"模型执行：{output_execution_summary.get('model_executed_count', 0)}", + f"记录样本：{output_execution_summary.get('recorded_fixture_count', 0)}", + f"Token 估算：{output_execution_summary.get('token_estimated_count', 0)}", + ] + else: + execution_items = ["尚未生成输出执行证据报告。"] capability_items = [ f"能力类型：{profile.get('task_family', 'Skill workflow')}", f"成熟度：{profile.get('maturity', 'scaffold')}", @@ -665,6 +696,9 @@ def render_html(summary: dict) -> str: gap: 16px; align-items: stretch; }} + .quality-panels {{ + margin-top: 16px; + }} .metrics-stack {{ display: grid; gap: 18px; @@ -1042,6 +1076,26 @@ def render_html(summary: dict) -> str: {bi_span("类型", "Type")}{bi_span("证据", "Evidence")}{bi_span("建议", "Action")} {quality_rows} +
    +
    +

    {bi_span("执行证据", "Execution Evidence")}

    + {render_list(execution_items)} +
    +
    +

    {bi_span("盲评审定", "Blind Adjudication")}

    + {render_list(review_items)} +
    +
    +
    +
    +

    {bi_span("评审原则", "Review Rule")}

    + {render_list(["先记录 reviewer 对 A/B 的选择，再打开答案 key 计算一致率。", "缺少真实 reviewer 决策时只显示待评审，不伪造人工结论。"])} +
    +
    +

    {bi_span("运行原则", "Run Rule")}

    + {render_list(["recorded fixture 只能证明可复现样本，不等同于模型执行。", "只有 provider runner 返回 model metadata 时才计入 model-executed。"])} +
    +
    diff --git a/scripts/render_social_preview.py b/scripts/render_social_preview.py index 237b113..7b23217 100644 --- a/scripts/render_social_preview.py +++ b/scripts/render_social_preview.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 from __future__ import annotations +import argparse from pathlib import Path from PIL import Image, ImageDraw, ImageFont @@ -123,6 +124,7 @@ def draw_png() -> None: def draw_svg() -> None: + OUT_DIR.mkdir(parents=True, exist_ok=True) svg = f""" YAO @@ -161,10 +163,21 @@ def draw_svg() -> None: def main() -> None: - draw_png() - draw_svg() - print(f"Wrote {PNG_PATH}") - print(f"Wrote {SVG_PATH}") + parser = argparse.ArgumentParser(description="Render the repository social preview image assets.") + parser.add_argument( + "--format", + choices=["all", "png", "svg"], + default="all", + help="Asset format to render.", + ) + args = parser.parse_args() + + if args.format in {"all", "png"}: + draw_png() + print(f"Wrote {PNG_PATH}") + if args.format in {"all", "svg"}: + draw_svg() + print(f"Wrote {SVG_PATH}") if __name__ == "__main__": diff --git a/scripts/run_conformance_suite.py b/scripts/run_conformance_suite.py index 21ab109..d51aa25 100644 --- a/scripts/run_conformance_suite.py +++ b/scripts/run_conformance_suite.py @@ -173,7 +173,7 @@ def check_target(skill_dir: Path, target: str, evidence: dict[str, Any]) -> dict if target in {"openai", "claude", "generic"}: add_check(checks, failures, bool(degradation.get(target)), f"{target} degradation note exists", f"Missing {target} degradation note") else: - warnings.append(f"{target} uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0.") + warnings.append(f"{target} uses canonical Agent Skills metadata; provider-native execution transforms are not implemented in v0.") add_check(checks, failures, ir.get("schema_version") == "2.0.0", "Skill IR schema_version is 2.0.0", "Missing or invalid Skill IR") add_check(checks, failures, ir.get("name") == name, "Skill IR name matches frontmatter", "Skill IR name does not match frontmatter") diff --git a/scripts/run_output_eval.py b/scripts/run_output_eval.py index 8d79f45..8183ceb 100644 --- a/scripts/run_output_eval.py +++ b/scripts/run_output_eval.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import hashlib import json from pathlib import Path from typing import Any @@ -7,6 +8,7 @@ from typing import Any ROOT = Path(__file__).resolve().parent.parent DEFAULT_CASES = ROOT / "evals" / "output" / "cases.jsonl" +BLIND_SEED = "yao-output-eval-blind-v1" def display_path(path: Path) -> str: @@ -115,6 +117,98 @@ def grade_case(case: dict[str, Any]) -> dict[str, Any]: } +def blind_variant_order(case_id: str) -> list[str]: + digest = hashlib.sha256(f"{BLIND_SEED}:{case_id}".encode("utf-8")).hexdigest() + return ["baseline", "with_skill"] if int(digest[:2], 16) % 2 == 0 else ["with_skill", "baseline"] + + +def output_for_role(case: dict[str, Any], role: str) -> str: + return str(case.get("baseline_output" if role == "baseline" else "with_skill_output", "")) + + +def expected_role(case: dict[str, Any]) -> str: + review = case.get("human_review", {}) if isinstance(case.get("human_review"), dict) else {} + winner = str(review.get("expected_winner", "with_skill")) + return winner if winner in {"baseline", "with_skill"} else "with_skill" + + +def build_blind_review_pack(cases: list[dict[str, Any]], results: list[dict[str, Any]]) -> tuple[dict[str, Any], dict[str, Any]]: + result_by_id = {item["id"]: item for item in results} + pairs = [] + answer_pairs = [] + for case in cases: + case_id = str(case["id"]) + order = blind_variant_order(case_id) + variant_a_role, variant_b_role = order + expected = expected_role(case) + expected_variant = "A" if variant_a_role == expected else "B" + assertions = case.get("assertions", []) if isinstance(case.get("assertions"), list) else [] + rubric = [ + { + "id": str(item.get("id", "assertion")), + "description": str(item.get("description", "")), + "weight": float(item.get("weight", 1) or 0), + } + for item in assertions + if isinstance(item, dict) + ] + pairs.append( + { + "case_id": case_id, + "prompt": str(case.get("prompt", "")), + "input_files": case.get("input_files", []), + "metadata": case.get("metadata", {}), + "review_instruction": "Pick A or B based only on the rubric. Do not infer which output came from the skill.", + "rubric": rubric, + "variant_a": { + "blind_id": f"{case_id}:A", + "output": output_for_role(case, variant_a_role), + }, + "variant_b": { + "blind_id": f"{case_id}:B", + "output": output_for_role(case, variant_b_role), + }, + } + ) + scored = result_by_id.get(case_id, {}) + answer_pairs.append( + { + "case_id": case_id, + "variant_a_role": variant_a_role, + "variant_b_role": variant_b_role, + "expected_winner_role": expected, + "expected_winner_variant": expected_variant, + "score_winner_role": scored.get("winner", ""), + "delta": scored.get("delta", 0), + } + ) + pack = { + "schema_version": "1.0", + "seed": BLIND_SEED, + "summary": { + "pair_count": len(pairs), + "answer_key_separate": True, + "with_skill_hidden_count": sum( + 1 + for pair in answer_pairs + if pair["variant_a_role"] == "with_skill" or pair["variant_b_role"] == "with_skill" + ), + }, + "pairs": pairs, + } + answer_key = { + "schema_version": "1.0", + "seed": BLIND_SEED, + "summary": { + "pair_count": len(answer_pairs), + "with_skill_expected_count": sum(1 for pair in answer_pairs if pair["expected_winner_role"] == "with_skill"), + "baseline_expected_count": sum(1 for pair in answer_pairs if pair["expected_winner_role"] == "baseline"), + }, + "answers": answer_pairs, + } + return pack, answer_key + + def build_summary(results: list[dict[str, Any]]) -> dict[str, Any]: case_count = len(results) baseline_average = sum(item["baseline"]["score"] for item in results) / case_count if case_count else 0 @@ -150,8 +244,12 @@ def render_markdown(payload: dict[str, Any]) -> str: f"- With-skill pass rate: `{summary['with_skill_pass_rate']}`", f"- Delta: `{summary['delta']}`", f"- Regressions: `{summary['regression_count']}`", + f"- Blind A/B pairs: `{summary.get('blind_pair_count', 0)}`", f"- Gate pass: `{summary['gate_pass']}`", "", + "Blind review artifacts are generated separately so reviewers can inspect A/B outputs without seeing the answer key.", + "Run output review adjudication after reviewer decisions are recorded; pending cases should stay pending rather than being counted as human agreement.", + "", "## Case Results", "", "| Case | Baseline | With Skill | Delta | Winner | Failed With-Skill Assertions |", @@ -181,10 +279,69 @@ def render_markdown(payload: dict[str, Any]) -> str: return "\n".join(lines).strip() + "\n" -def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dict[str, Any]: +def render_blind_review_markdown(pack: dict[str, Any]) -> str: + summary = pack["summary"] + lines = [ + "# Output Blind A/B Review Pack", + "", + "This packet hides whether each variant came from the baseline or the skill-guided output. Use the separate answer key only after review.", + "", + f"- Pairs: `{summary['pair_count']}`", + f"- Seed: `{pack['seed']}`", + f"- Answer key separate: `{summary['answer_key_separate']}`", + "", + ] + for pair in pack["pairs"]: + lines.extend( + [ + f"## Case: {pair['case_id']}", + "", + f"Prompt: {pair['prompt']}", + "", + "Rubric:", + ] + ) + for item in pair["rubric"]: + lines.append(f"- `{item['id']}` ({item['weight']}): {item['description']}") + lines.extend( + [ + "", + "### Variant A", + "", + str(pair["variant_a"]["output"]), + "", + "### Variant B", + "", + str(pair["variant_b"]["output"]), + "", + ] + ) + return "\n".join(lines).strip() + "\n" + + +def run_output_eval( + cases_path: Path, + output_json: Path, + output_md: Path, + blind_pack_json: Path, + blind_pack_md: Path, + blind_answer_key_json: Path, +) -> dict[str, Any]: cases = load_cases(cases_path) validation_failures = [failure for case in cases for failure in validate_case(case, cases_path.parent)] if validation_failures: + blind_pack = { + "schema_version": "1.0", + "seed": BLIND_SEED, + "summary": {"pair_count": 0, "answer_key_separate": True, "with_skill_hidden_count": 0}, + "pairs": [], + } + blind_answer_key = { + "schema_version": "1.0", + "seed": BLIND_SEED, + "summary": {"pair_count": 0, "with_skill_expected_count": 0, "baseline_expected_count": 0}, + "answers": [], + } payload = { "ok": False, "cases": display_path(cases_path), @@ -195,6 +352,7 @@ def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dic "delta": 0, "regression_count": 0, "gate_pass": False, + "blind_pair_count": 0, "failure_taxonomy": ["invalid_case"], }, "results": [], @@ -202,6 +360,7 @@ def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dic } else: results = [grade_case(case) for case in cases] + blind_pack, blind_answer_key = build_blind_review_pack(cases, results) payload = { "ok": True, "cases": display_path(cases_path), @@ -209,11 +368,29 @@ def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dic "results": results, "failures": [], } - payload["artifacts"] = {"json": display_path(output_json), "markdown": display_path(output_md)} + payload["summary"]["blind_pair_count"] = blind_pack["summary"]["pair_count"] + payload["blind_review"] = { + "pack": display_path(blind_pack_json), + "answer_key": display_path(blind_answer_key_json), + "pair_count": blind_pack["summary"]["pair_count"], + } + payload["artifacts"] = { + "json": display_path(output_json), + "markdown": display_path(output_md), + "blind_review_pack_json": display_path(blind_pack_json), + "blind_review_pack_md": display_path(blind_pack_md), + "blind_answer_key_json": display_path(blind_answer_key_json), + } output_json.parent.mkdir(parents=True, exist_ok=True) output_md.parent.mkdir(parents=True, exist_ok=True) + blind_pack_json.parent.mkdir(parents=True, exist_ok=True) + blind_pack_md.parent.mkdir(parents=True, exist_ok=True) + blind_answer_key_json.parent.mkdir(parents=True, exist_ok=True) output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") output_md.write_text(render_markdown(payload), encoding="utf-8") + blind_pack_json.write_text(json.dumps(blind_pack, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + blind_pack_md.write_text(render_blind_review_markdown(blind_pack), encoding="utf-8") + blind_answer_key_json.write_text(json.dumps(blind_answer_key, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") return payload @@ -222,12 +399,18 @@ def main() -> None: parser.add_argument("--cases", default=str(DEFAULT_CASES)) parser.add_argument("--output-json", default=str(ROOT / "reports" / "output_quality_scorecard.json")) parser.add_argument("--output-md", default=str(ROOT / "reports" / "output_quality_scorecard.md")) + parser.add_argument("--blind-pack-json", default=str(ROOT / "reports" / "output_blind_review_pack.json")) + parser.add_argument("--blind-pack-md", default=str(ROOT / "reports" / "output_blind_review_pack.md")) + parser.add_argument("--blind-answer-key-json", default=str(ROOT / "reports" / "output_blind_answer_key.json")) args = parser.parse_args() payload = run_output_eval( Path(args.cases).resolve(), Path(args.output_json).resolve(), Path(args.output_md).resolve(), + Path(args.blind_pack_json).resolve(), + Path(args.blind_pack_md).resolve(), + Path(args.blind_answer_key_json).resolve(), ) print(json.dumps(payload, ensure_ascii=False, indent=2)) raise SystemExit(0 if payload["ok"] else 2) diff --git a/scripts/run_output_execution.py b/scripts/run_output_execution.py new file mode 100644 index 0000000..ad0a97a --- /dev/null +++ b/scripts/run_output_execution.py @@ -0,0 +1,382 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import shlex +import subprocess +import time +from pathlib import Path +from typing import Any + +from run_output_eval import DEFAULT_CASES, display_path, grade_output, load_cases, validate_case + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_execution_runs.json" +DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_execution_runs.md" +VARIANTS = ("baseline", "with_skill") + + +def output_for_variant(case: dict[str, Any], variant: str) -> str: + return str(case.get("baseline_output" if variant == "baseline" else "with_skill_output", "")) + + +def output_key(variant: str) -> str: + return "baseline_output" if variant == "baseline" else "with_skill_output" + + +def parse_runner_command(value: str | None) -> list[str]: + if not value: + return [] + stripped = value.strip() + if stripped.startswith("["): + payload = json.loads(stripped) + if not isinstance(payload, list) or not all(isinstance(item, str) and item for item in payload): + raise ValueError("--runner-command JSON must be a non-empty string list") + return payload + return shlex.split(stripped) + + +def display_command(command: list[str]) -> list[str]: + displayed: list[str] = [] + for item in command: + path = Path(item) + if path.is_absolute() and path.exists(): + displayed.append(display_path(path)) + else: + displayed.append(item) + return displayed + + +def estimate_tokens(text: str) -> int: + if not text: + return 0 + return max(1, round(len(text) / 4)) + + +def parse_runner_stdout(stdout: str) -> tuple[str, dict[str, Any]]: + stripped = stdout.strip() + if not stripped: + return "", {} + try: + payload = json.loads(stripped) + except json.JSONDecodeError: + return stdout, {} + if not isinstance(payload, dict): + return stdout, {} + return str(payload.get("output", "")), payload + + +def usage_payload(payload: dict[str, Any], prompt: str, output: str) -> dict[str, Any]: + usage = payload.get("usage", {}) if isinstance(payload.get("usage"), dict) else {} + input_tokens = usage.get("input_tokens", usage.get("prompt_tokens")) + output_tokens = usage.get("output_tokens", usage.get("completion_tokens")) + total_tokens = usage.get("total_tokens") + estimated = bool(usage.get("estimated", False)) + if input_tokens is None: + input_tokens = estimate_tokens(prompt) + estimated = True + if output_tokens is None: + output_tokens = estimate_tokens(output) + estimated = True + if total_tokens is None: + total_tokens = int(input_tokens or 0) + int(output_tokens or 0) + estimated = True + return { + "input_tokens": int(input_tokens or 0), + "output_tokens": int(output_tokens or 0), + "total_tokens": int(total_tokens or 0), + "estimated": estimated, + } + + +def recorded_fixture_run(case: dict[str, Any], variant: str, assertions: list[dict[str, Any]]) -> dict[str, Any]: + output = output_for_variant(case, variant) + grade = grade_output(output, assertions) + return { + "case_id": str(case.get("id", "")), + "variant": variant, + "status": "pass", + "execution_mode": "recorded_fixture", + "model_executed": False, + "command_executed": False, + "duration_ms": None, + "provider": "", + "model": "", + "usage": usage_payload({}, str(case.get("prompt", "")), output), + "score": grade["score"], + "passed_count": grade["passed_count"], + "failed_count": grade["failed_count"], + "failed_assertions": [item["id"] for item in grade["failed"]], + "output_sha256": hashlib.sha256(output.encode("utf-8")).hexdigest(), + "failure": "", + } + + +def command_run( + case: dict[str, Any], + variant: str, + assertions: list[dict[str, Any]], + command: list[str], + timeout_seconds: float, +) -> dict[str, Any]: + request = { + "case_id": str(case.get("id", "")), + "variant": variant, + "prompt": str(case.get("prompt", "")), + "input_files": case.get("input_files", []), + "metadata": case.get("metadata", {}), + "fixture_output": output_for_variant(case, variant), + "output_key": output_key(variant), + } + started = time.perf_counter() + try: + proc = subprocess.run( + command, + input=json.dumps(request, ensure_ascii=False), + capture_output=True, + text=True, + timeout=timeout_seconds, + check=False, + ) + duration_ms = round((time.perf_counter() - started) * 1000, 2) + except subprocess.TimeoutExpired as exc: + return { + "case_id": request["case_id"], + "variant": variant, + "status": "fail", + "execution_mode": "command", + "model_executed": False, + "command_executed": True, + "duration_ms": round(timeout_seconds * 1000, 2), + "provider": "", + "model": "", + "usage": usage_payload({}, request["prompt"], ""), + "score": 0, + "passed_count": 0, + "failed_count": len(assertions), + "failed_assertions": [str(item.get("id", "assertion")) for item in assertions], + "output_sha256": "", + "failure": f"runner timed out after {timeout_seconds}s: {exc}", + } + output, payload = parse_runner_stdout(proc.stdout) + grade = grade_output(output, assertions) + execution_kind = str(payload.get("execution_kind", "command")) + provider = str(payload.get("provider", "")) + model = str(payload.get("model", "")) + model_executed = execution_kind == "model" or bool(model and provider) + return { + "case_id": request["case_id"], + "variant": variant, + "status": "pass" if proc.returncode == 0 and output else "fail", + "execution_mode": execution_kind if execution_kind in {"command", "model"} else "command", + "model_executed": model_executed, + "command_executed": True, + "duration_ms": duration_ms, + "provider": provider, + "model": model, + "usage": usage_payload(payload, request["prompt"], output), + "score": grade["score"] if output else 0, + "passed_count": grade["passed_count"] if output else 0, + "failed_count": grade["failed_count"] if output else len(assertions), + "failed_assertions": [item["id"] for item in grade["failed"]] if output else [str(item.get("id", "assertion")) for item in assertions], + "output_sha256": hashlib.sha256(output.encode("utf-8")).hexdigest() if output else "", + "failure": "" if proc.returncode == 0 and output else (proc.stderr.strip() or "runner returned no output"), + } + + +def build_summary(runs: list[dict[str, Any]]) -> dict[str, Any]: + variant_run_count = len(runs) + case_ids = sorted({item["case_id"] for item in runs}) + baseline = [item for item in runs if item["variant"] == "baseline"] + with_skill = [item for item in runs if item["variant"] == "with_skill"] + baseline_average = sum(item["score"] for item in baseline) / len(baseline) if baseline else 0 + with_skill_average = sum(item["score"] for item in with_skill) / len(with_skill) if with_skill else 0 + regression_count = sum( + 1 + for case_id in case_ids + if max((item["score"] for item in with_skill if item["case_id"] == case_id), default=0) + < max((item["score"] for item in baseline if item["case_id"] == case_id), default=0) + ) + failure_count = sum(1 for item in runs if item["status"] != "pass") + command_executed_count = sum(1 for item in runs if item.get("command_executed")) + model_executed_count = sum(1 for item in runs if item.get("model_executed")) + recorded_fixture_count = sum(1 for item in runs if item.get("execution_mode") == "recorded_fixture") + timing_observed_count = sum(1 for item in runs if item.get("duration_ms") is not None) + token_estimated_count = sum(1 for item in runs if item.get("usage", {}).get("estimated")) + token_observed_count = variant_run_count - token_estimated_count + return { + "case_count": len(case_ids), + "variant_run_count": variant_run_count, + "command_executed_count": command_executed_count, + "model_executed_count": model_executed_count, + "recorded_fixture_count": recorded_fixture_count, + "timing_observed_count": timing_observed_count, + "token_observed_count": token_observed_count, + "token_estimated_count": token_estimated_count, + "baseline_pass_rate": round(baseline_average, 2), + "with_skill_pass_rate": round(with_skill_average, 2), + "delta": round(with_skill_average - baseline_average, 2), + "regression_count": regression_count, + "failure_count": failure_count, + "gate_pass": failure_count == 0 and with_skill_average >= baseline_average and regression_count == 0, + } + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + "# Output Execution Runs", + "", + "This report records how output-eval variants were produced and whether timing or token evidence is observed or estimated.", + "", + f"- Cases: `{summary['case_count']}`", + f"- Variant runs: `{summary['variant_run_count']}`", + f"- Command executed: `{summary['command_executed_count']}`", + f"- Model executed: `{summary['model_executed_count']}`", + f"- Recorded fixtures: `{summary['recorded_fixture_count']}`", + f"- Timing observed: `{summary['timing_observed_count']}`", + f"- Token observed: `{summary['token_observed_count']}`", + f"- Token estimated: `{summary['token_estimated_count']}`", + f"- Delta: `{summary['delta']}`", + f"- Gate pass: `{summary['gate_pass']}`", + "", + ] + if summary["model_executed_count"] == 0: + lines.extend( + [ + "No model-executed runs are recorded yet.", + "", + "Use `--runner-command` with a provider-backed runner to replace recorded fixtures with real model output evidence.", + "", + ] + ) + if summary["command_executed_count"] > 0: + lines.extend( + [ + "Command runner evidence is present. This proves the eval harness executed an external command, but it is not provider-backed model evidence unless the runner reports model metadata.", + "", + ] + ) + lines.extend( + [ + "## Runs", + "", + "| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |", + "| --- | --- | --- | --- | ---: | ---: | ---: | --- |", + ] + ) + for item in payload["runs"]: + usage = item.get("usage", {}) + duration = "" if item.get("duration_ms") is None else str(item["duration_ms"]) + model = item.get("model") or item.get("provider") or "" + lines.append( + f"| {item['case_id']} | {item['variant']} | {item['execution_mode']} | {model} | " + f"{duration} | {usage.get('total_tokens', 0)} | {item['score']} | {item['status']} |" + ) + failures = [item for item in payload["runs"] if item.get("failure")] + if failures: + lines.extend(["", "## Failures", ""]) + for item in failures: + lines.append(f"- `{item['case_id']}` `{item['variant']}`: {item['failure']}") + lines.extend( + [ + "", + "## Next Fixes", + "", + "- Keep recorded fixtures as reproducible baselines, but do not describe them as model-executed evidence.", + "- Add provider-backed command runners for holdout cases when release confidence depends on real generation behavior.", + "- Compare timing, token cost, and assertion deltas before promoting a skill to governed reuse.", + ] + ) + return "\n".join(lines).strip() + "\n" + + +def run_output_execution( + cases_path: Path, + output_json: Path, + output_md: Path, + runner_command: list[str], + timeout_seconds: float, +) -> dict[str, Any]: + cases = load_cases(cases_path) + validation_failures = [failure for case in cases for failure in validate_case(case, cases_path.parent)] + runs: list[dict[str, Any]] = [] + if not validation_failures: + for case in cases: + assertions = case.get("assertions", []) if isinstance(case.get("assertions"), list) else [] + for variant in VARIANTS: + if runner_command: + runs.append(command_run(case, variant, assertions, runner_command, timeout_seconds)) + else: + runs.append(recorded_fixture_run(case, variant, assertions)) + summary = build_summary(runs) + failures = validation_failures + [f"{item['case_id']} {item['variant']}: {item['failure']}" for item in runs if item.get("failure")] + payload = { + "schema_version": "1.0", + "ok": not failures and summary["gate_pass"], + "cases": display_path(cases_path), + "runner": { + "mode": "command" if runner_command else "recorded_fixture", + "command": display_command(runner_command), + "timeout_seconds": timeout_seconds if runner_command else None, + }, + "summary": summary, + "runs": runs, + "failures": failures, + "artifacts": { + "json": display_path(output_json), + "markdown": display_path(output_md), + }, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Record output-eval execution evidence for static, command, or model-backed runs.") + parser.add_argument("--cases", default=str(DEFAULT_CASES)) + parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON)) + parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD)) + parser.add_argument("--runner-command", help="Command string or JSON string list. Receives a JSON request on stdin.") + parser.add_argument("--timeout-seconds", type=float, default=30.0) + args = parser.parse_args() + + try: + runner_command = parse_runner_command(args.runner_command) + except (json.JSONDecodeError, ValueError) as exc: + payload = { + "schema_version": "1.0", + "ok": False, + "cases": display_path(Path(args.cases).resolve()), + "runner": {"mode": "invalid", "command": [], "timeout_seconds": args.timeout_seconds}, + "summary": build_summary([]), + "runs": [], + "failures": [str(exc)], + "artifacts": {"json": display_path(Path(args.output_json).resolve()), "markdown": display_path(Path(args.output_md).resolve())}, + } + output_json = Path(args.output_json).resolve() + output_md = Path(args.output_md).resolve() + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(2) + + payload = run_output_execution( + Path(args.cases).resolve(), + Path(args.output_json).resolve(), + Path(args.output_md).resolve(), + runner_command, + args.timeout_seconds, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/simulate_install.py b/scripts/simulate_install.py new file mode 100644 index 0000000..03cc92c --- /dev/null +++ b/scripts/simulate_install.py @@ -0,0 +1,271 @@ +#!/usr/bin/env python3 +import argparse +import json +import shutil +import tempfile +import zipfile +from datetime import date +from pathlib import Path, PurePosixPath +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent + + +def display_path(path: Path, root: Path = ROOT) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def read_frontmatter(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + text = path.read_text(encoding="utf-8", errors="replace") + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + if yaml is not None: + payload = yaml.safe_load(parts[1]) or {} + return payload if isinstance(payload, dict) else {} + data: dict[str, Any] = {} + for line in parts[1].splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data + + +def zip_names(zip_path: Path) -> list[str]: + with zipfile.ZipFile(zip_path) as archive: + return archive.namelist() + + +def unsafe_zip_entries(names: list[str]) -> list[str]: + unsafe = [] + for name in names: + pure = PurePosixPath(name) + if pure.is_absolute() or ".." in pure.parts or name.startswith("\\") or (pure.parts and ":" in pure.parts[0]): + unsafe.append(name) + return unsafe + + +def top_level_dirs(names: list[str]) -> list[str]: + roots = set() + for name in names: + parts = PurePosixPath(name).parts + if parts: + roots.add(parts[0]) + return sorted(roots) + + +def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str, passed: bool, detail: str) -> None: + checks.append({"id": check_id, "status": "pass" if passed else "fail", "detail": detail}) + if not passed: + failures.append(detail) + + +def adapter_targets(package_dir: Path) -> list[str]: + targets_dir = package_dir / "targets" + if not targets_dir.exists(): + return [] + return sorted(path.name for path in targets_dir.iterdir() if path.is_dir()) + + +def simulate_install(skill_dir: Path, package_dir: Path, install_root: Path | None, generated_at: str) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + package_dir = package_dir.resolve() + checks: list[dict[str, str]] = [] + failures: list[str] = [] + warnings: list[str] = [] + archive_path = package_dir / f"{skill_dir.name}.zip" + package_manifest = load_json(package_dir / "manifest.json") + installed_dir = Path("") + archive_entries: list[str] = [] + + temp_dir: tempfile.TemporaryDirectory[str] | None = None + if install_root is None: + temp_dir = tempfile.TemporaryDirectory(prefix="yao-skill-install-") + install_base = Path(temp_dir.name) + install_root_is_temp = True + else: + requested_root = install_root.resolve() + requested_root.mkdir(parents=True, exist_ok=True) + install_base = requested_root / f"simulate-{skill_dir.name}" + if install_base.exists(): + shutil.rmtree(install_base) + install_base.mkdir(parents=True, exist_ok=True) + install_root_is_temp = False + + try: + add_check(checks, failures, "archive-present", archive_path.exists(), f"Package archive exists: {display_path(archive_path)}") + if archive_path.exists(): + try: + archive_entries = zip_names(archive_path) + except zipfile.BadZipFile: + add_check(checks, failures, "archive-readable", False, f"Archive is not a readable zip: {display_path(archive_path)}") + else: + unsafe_entries = unsafe_zip_entries(archive_entries) + add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries") + roots = top_level_dirs(archive_entries) + add_check(checks, failures, "single-top-level", roots == [skill_dir.name], f"Archive top-level directory is {skill_dir.name}") + if not unsafe_entries and roots: + with zipfile.ZipFile(archive_path) as archive: + archive.extractall(install_base) + installed_dir = install_base / roots[0] + + frontmatter = read_frontmatter(installed_dir / "SKILL.md") if installed_dir else {} + source_manifest = load_json(installed_dir / "manifest.json") if installed_dir else {} + interface_doc = load_yaml(installed_dir / "agents" / "interface.yaml") if installed_dir else {} + add_check(checks, failures, "entrypoint-load", bool(frontmatter), "Installed SKILL.md frontmatter is readable") + add_check(checks, failures, "entrypoint-name", frontmatter.get("name") == skill_dir.name, "Installed SKILL.md name matches package directory") + add_check(checks, failures, "entrypoint-description", bool(frontmatter.get("description")), "Installed SKILL.md description is present") + add_check(checks, failures, "manifest-load", bool(source_manifest), "Installed manifest.json is readable") + add_check(checks, failures, "manifest-name", source_manifest.get("name") == package_manifest.get("name"), "Installed manifest name matches package manifest") + add_check(checks, failures, "manifest-version", source_manifest.get("version") == package_manifest.get("version"), "Installed manifest version matches package manifest") + add_check(checks, failures, "interface-load", bool(interface_doc.get("interface")), "Installed agents/interface.yaml is readable") + add_check(checks, failures, "overview-report", (installed_dir / "reports" / "skill-overview.html").exists(), "Installed overview report is present") + add_check(checks, failures, "review-studio-report", (installed_dir / "reports" / "review-studio.html").exists(), "Installed Review Studio report is present") + + adapters = adapter_targets(package_dir) + for target in adapters: + adapter = load_json(package_dir / "targets" / target / "adapter.json") + add_check(checks, failures, f"adapter-{target}", bool(adapter), f"{target} adapter is readable after package install simulation") + add_check( + checks, + failures, + f"adapter-{target}-name", + adapter.get("name") == package_manifest.get("name"), + f"{target} adapter name matches package manifest", + ) + + if not adapters: + warnings.append("No target adapters found in package directory.") + + if install_root_is_temp: + install_root_display = "[temporary-install-root]" + installed_dir_display = f"{install_root_display}/{installed_dir.name}" if installed_dir else "" + else: + install_root_display = display_path(install_base) + installed_dir_display = display_path(installed_dir) if installed_dir else "" + + report = { + "ok": not failures, + "schema_version": "2.0", + "generated_at": generated_at, + "skill_dir": display_path(skill_dir), + "package_dir": display_path(package_dir), + "install_root": install_root_display, + "installed_skill_dir": installed_dir_display, + "summary": { + "archive_present": archive_path.exists(), + "archive_entry_count": len(archive_entries), + "archive_extracted": bool(installed_dir and installed_dir.exists()), + "entrypoint_loaded": bool(frontmatter), + "manifest_loaded": bool(source_manifest), + "interface_loaded": bool(interface_doc.get("interface")), + "adapter_count": len(adapters), + "install_root_is_temp": install_root_is_temp, + "failure_count": len(failures), + "warning_count": len(warnings), + }, + "checks": checks, + "failures": failures, + "warnings": warnings, + "artifacts": { + "archive": display_path(archive_path) if archive_path.exists() else "", + "package_manifest": display_path(package_dir / "manifest.json"), + }, + } + return report + finally: + if temp_dir is not None: + temp_dir.cleanup() + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Install Simulation", + "", + f"- OK: `{report['ok']}`", + f"- Package directory: `{report['package_dir']}`", + f"- Archive extracted: `{summary['archive_extracted']}`", + f"- Entrypoint loaded: `{summary['entrypoint_loaded']}`", + f"- Manifest loaded: `{summary['manifest_loaded']}`", + f"- Interface loaded: `{summary['interface_loaded']}`", + f"- Adapters readable: `{summary['adapter_count']}`", + f"- Failures: `{summary['failure_count']}`", + f"- Warnings: `{summary['warning_count']}`", + "", + "## Checks", + "", + "| Check | Status | Detail |", + "| --- | --- | --- |", + ] + for item in report["checks"]: + lines.append(f"| `{item['id']}` | `{item['status']}` | {item['detail']} |") + lines.extend(["", "## Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) + lines.extend(["", "## Warnings", ""]) + lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"]) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Simulate installing a generated skill package into a temporary local skill root.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--package-dir", default="dist") + parser.add_argument("--install-root") + parser.add_argument("--output-json", default="reports/install_simulation.json") + parser.add_argument("--output-md", default="reports/install_simulation.md") + parser.add_argument("--generated-at", default=str(date.today())) + args = parser.parse_args() + + package_dir = Path(args.package_dir) + if not package_dir.is_absolute(): + package_dir = Path.cwd() / package_dir + install_root = Path(args.install_root) if args.install_root else None + report = simulate_install(Path(args.skill_dir), package_dir, install_root, args.generated_at) + + output_json = Path(args.output_json) + output_md = Path(args.output_md) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + report["artifacts"]["json"] = display_path(output_json) + report["artifacts"]["markdown"] = display_path(output_md) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/skill_report_charts.py b/scripts/skill_report_charts.py index c34f792..69d738b 100644 --- a/scripts/skill_report_charts.py +++ b/scripts/skill_report_charts.py @@ -3,6 +3,9 @@ import html import math +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to render inline SVG report charts." + BRAND = "#1B365D" BORDER = "#e8e6dc" SOFT = "#faf9f5" diff --git a/scripts/skill_report_metrics.py b/scripts/skill_report_metrics.py index 2a894a2..9ec3cb9 100644 --- a/scripts/skill_report_metrics.py +++ b/scripts/skill_report_metrics.py @@ -2,8 +2,12 @@ from pathlib import Path +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by skill_report_model.py to calculate overview report metrics." + REPORT_EVIDENCE = [ "skill-ir.json", + "compiled_targets.json", "intent-dialogue.json", "intent-confidence.json", "reference-synthesis.json", @@ -11,6 +15,12 @@ REPORT_EVIDENCE = [ "conformance_matrix.json", "security_trust_report.json", "skill_atlas.json", + "registry_audit.json", + "package_verification.json", + "install_simulation.json", + "upgrade_check.json", + "adoption_drift_report.json", + "review_waivers.json", "artifact-design-profile.json", "prompt-quality-profile.json", "system-model.json", diff --git a/scripts/skill_report_model.py b/scripts/skill_report_model.py index afdb36d..5e9b86d 100644 --- a/scripts/skill_report_model.py +++ b/scripts/skill_report_model.py @@ -12,6 +12,9 @@ except ImportError: # pragma: no cover yaml = None +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by render_skill_overview.py to build the v2 report data model." + KNOWN_ENTRIES = [ ("SKILL.md", "Skill entrypoint"), ("README.md", "Human-readable usage guide"), @@ -194,14 +197,30 @@ def derive_strengths(skill_dir: Path, metadata: dict) -> list[str]: strengths = ["触发面保持精简，并锚定在 frontmatter description。"] if (skill_dir / "reports" / "skill-ir.json").exists() or (skill_dir / "skill-ir" / "examples").exists(): strengths.append("已生成 Skill IR，核心语义可先于平台打包被审查和迁移。") + if (skill_dir / "reports" / "compiled_targets.json").exists(): + strengths.append("已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。") if (skill_dir / "reports" / "output_quality_scorecard.json").exists(): strengths.append("已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。") + if (skill_dir / "reports" / "output_execution_runs.json").exists(): + strengths.append("已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。") + if (skill_dir / "reports" / "output_review_adjudication.json").exists(): + strengths.append("已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。") if (skill_dir / "reports" / "conformance_matrix.json").exists(): strengths.append("已生成 Runtime Conformance Matrix，可审查目标端消费能力。") if (skill_dir / "reports" / "security_trust_report.json").exists(): strengths.append("已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。") if (skill_dir / "reports" / "skill_atlas.json").exists(): strengths.append("已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。") + if (skill_dir / "reports" / "registry_audit.json").exists(): + strengths.append("已生成 Registry Audit，可审查版本、owner、license、checksum 和目标兼容矩阵。") + if (skill_dir / "reports" / "install_simulation.json").exists(): + strengths.append("已生成 Install Simulation，可审查 zip 解压、入口加载、接口元数据和 adapter 可读性。") + if (skill_dir / "reports" / "adoption_drift_report.json").exists(): + strengths.append("已生成 Adoption Drift Report，可把本地使用反馈转为下一轮迭代信号。") + if (skill_dir / "reports" / "review_waivers.json").exists(): + strengths.append("已生成 Review Waivers 台账，可记录 reviewer 对 warning 风险的批准、理由和到期时间。") + if (skill_dir / "reports" / "review_annotations.json").exists(): + strengths.append("已生成 Review Annotations 台账，可把 reviewer 批注挂到 gate、文件和行号。") if (skill_dir / "reports" / "review-studio.json").exists(): strengths.append("已生成 Review Studio 2.0，可在一页中查看 blocker、warning、证据路径和发布闸门。") if (skill_dir / "agents" / "interface.yaml").exists(): @@ -454,9 +473,21 @@ def build_report_model(skill_dir: Path) -> dict: system_model = load_json(skill_dir / "reports" / "system-model.json") output_risk = load_json(skill_dir / "reports" / "output-risk-profile.json") output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json") + output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json") + output_blind_review = load_json(skill_dir / "reports" / "output_blind_review_pack.json") + output_review_adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json") conformance = load_json(skill_dir / "reports" / "conformance_matrix.json") + runtime_permissions = load_json(skill_dir / "reports" / "runtime_permission_probes.json") trust_report = load_json(skill_dir / "reports" / "security_trust_report.json") skill_atlas = load_json(skill_dir / "reports" / "skill_atlas.json") + registry_audit = load_json(skill_dir / "reports" / "registry_audit.json") + package_verification = load_json(skill_dir / "reports" / "package_verification.json") + install_simulation = load_json(skill_dir / "reports" / "install_simulation.json") + upgrade_check = load_json(skill_dir / "reports" / "upgrade_check.json") + adoption_drift = load_json(skill_dir / "reports" / "adoption_drift_report.json") + review_waivers = load_json(skill_dir / "reports" / "review_waivers.json") + review_annotations = load_json(skill_dir / "reports" / "review_annotations.json") + compiled_targets = load_json(skill_dir / "reports" / "compiled_targets.json") skill_ir = load_json(skill_dir / "reports" / "skill-ir.json") if not skill_ir: example_ir = skill_dir / "skill-ir" / "examples" / f"{frontmatter.get('name', skill_dir.name)}.json" @@ -485,6 +516,41 @@ def build_report_model(skill_dir: Path) -> dict: "skill_archetype": manifest.get("skill_archetype", manifest.get("maturity_tier", "scaffold")), "updated_at": manifest.get("updated_at", str(date.today())), } + deliverables = [ + "SKILL.md", + "agents/interface.yaml", + "reports/skill-ir.json", + "reports/compiled_targets.md", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/skill_atlas.html", + "reports/registry_audit.md", + "reports/package_verification.md", + "reports/install_simulation.md", + "reports/upgrade_check.md", + "reports/adoption_drift_report.md", + "reports/review_waivers.md", + "reports/review_annotations.md", + "reports/review-studio.html", + "reports/skill-overview.html", + ] + if (skill_dir / "reports" / "runtime_permission_probes.md").exists(): + insert_after = deliverables.index("reports/security_trust_report.md") + 1 + deliverables.insert(insert_after, "reports/runtime_permission_probes.md") + if (skill_dir / "reports" / "output_blind_review_pack.md").exists(): + insert_after = deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/output_blind_review_pack.md") + if (skill_dir / "reports" / "output_execution_runs.md").exists(): + insert_after = deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/output_execution_runs.md") + if (skill_dir / "reports" / "output_blind_answer_key.json").exists(): + insert_after = deliverables.index("reports/output_blind_review_pack.md") + 1 if "reports/output_blind_review_pack.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/output_blind_answer_key.json") + if (skill_dir / "reports" / "output_review_adjudication.md").exists(): + insert_after = deliverables.index("reports/output_blind_answer_key.json") + 1 if "reports/output_blind_answer_key.json" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/output_review_adjudication.md") + skill_summary = { "name": name, "title": title, @@ -494,17 +560,7 @@ def build_report_model(skill_dir: Path) -> dict: "updated_at": metadata["updated_at"], "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", "audience": "Skill 作者、复用团队和后续 reviewer。", - "deliverables": [ - "SKILL.md", - "agents/interface.yaml", - "reports/skill-ir.json", - "reports/output_quality_scorecard.md", - "reports/conformance_matrix.md", - "reports/security_trust_report.md", - "reports/skill_atlas.html", - "reports/review-studio.html", - "reports/skill-overview.html", - ], + "deliverables": deliverables, "flow": ["输入材料", "Skill 包体", "可复用能力"], } contract = { @@ -573,10 +629,115 @@ def build_report_model(skill_dir: Path) -> dict: "trigger_samples": len(skill_ir.get("trigger_surface", {}).get("should_trigger", [])), "output_eval_cases": len(skill_ir.get("eval_plan", {}).get("output", [])), }, + "compiled_targets": { + "ok": compiled_targets.get("ok", False), + "schema_version": compiled_targets.get("schema_version", ""), + "summary": compiled_targets.get("summary", {}), + "targets": [ + { + "target": item.get("target", ""), + "status": item.get("status", ""), + "adapter_mode": item.get("target_transform", {}).get("adapter_mode", ""), + "degradation_strategy": item.get("target_transform", {}).get("degradation_strategy", ""), + "native_surface": item.get("target_native_contract", {}).get("native_surface", ""), + "permission_enforcement": item.get("target_native_contract", {}).get("permissions", {}).get("enforcement", ""), + "generated_files": item.get("target_transform", {}).get("generated_files", []), + "unsupported_features": item.get("unsupported_features", []), + "warnings": item.get("warnings", []), + } + for item in compiled_targets.get("targets", []) + if isinstance(item, dict) + ], + "failures": compiled_targets.get("failures", []), + "warnings": compiled_targets.get("warnings", []), + }, "output_quality": output_quality.get("summary", {}), + "output_execution": { + "ok": output_execution.get("ok", False), + "summary": output_execution.get("summary", {}), + "runner": output_execution.get("runner", {}), + "failures": output_execution.get("failures", []), + }, + "output_blind_review": { + "summary": output_blind_review.get("summary", {}), + "seed": output_blind_review.get("seed", ""), + "pair_count": output_blind_review.get("summary", {}).get("pair_count", 0), + "answer_key_separate": output_blind_review.get("summary", {}).get("answer_key_separate", False), + }, + "output_review_adjudication": { + "ok": output_review_adjudication.get("ok", False), + "summary": output_review_adjudication.get("summary", {}), + "reviewer": output_review_adjudication.get("reviewer", ""), + "reviewed_at": output_review_adjudication.get("reviewed_at", ""), + "failures": output_review_adjudication.get("failures", []), + }, "runtime_conformance": conformance.get("summary", {}), + "runtime_permissions": { + "ok": runtime_permissions.get("ok", False), + "summary": runtime_permissions.get("summary", {}), + "expected_capabilities": runtime_permissions.get("expected_capabilities", []), + "targets": [ + { + "target": item.get("target", ""), + "status": item.get("status", ""), + "assurance": item.get("assurance", ""), + "native_enforcement": item.get("native_enforcement"), + "metadata_fallback_explicit": item.get("metadata_fallback_explicit", False), + "residual_risks": item.get("residual_risks", []), + } + for item in runtime_permissions.get("targets", []) + if isinstance(item, dict) + ], + "failures": runtime_permissions.get("failures", []), + }, "trust_security": trust_report.get("summary", {}), "skill_atlas": skill_atlas.get("summary", {}), + "registry_distribution": { + "ok": registry_audit.get("ok", False), + "package": registry_audit.get("package", {}), + "failures": registry_audit.get("failures", []), + "warnings": registry_audit.get("warnings", []), + }, + "package_verification": { + "ok": package_verification.get("ok", False), + "summary": package_verification.get("summary", {}), + "failures": package_verification.get("failures", []), + "warnings": package_verification.get("warnings", []), + }, + "install_simulation": { + "ok": install_simulation.get("ok", False), + "summary": install_simulation.get("summary", {}), + "failures": install_simulation.get("failures", []), + "warnings": install_simulation.get("warnings", []), + }, + "upgrade_check": { + "ok": upgrade_check.get("ok", False), + "summary": upgrade_check.get("summary", {}), + "upgrade_diff": upgrade_check.get("upgrade_diff", {}), + "release_notes": upgrade_check.get("release_notes", []), + "failures": upgrade_check.get("failures", []), + "warnings": upgrade_check.get("warnings", []), + }, + "adoption_drift": { + "ok": adoption_drift.get("ok", False), + "summary": adoption_drift.get("summary", {}), + "next_iteration_candidates": adoption_drift.get("next_iteration_candidates", []), + "privacy_contract": adoption_drift.get("privacy_contract", {}), + "failures": adoption_drift.get("failures", []), + }, + "review_waivers": { + "ok": review_waivers.get("ok", False), + "summary": review_waivers.get("summary", {}), + "policy": review_waivers.get("policy", {}), + "failures": review_waivers.get("failures", []), + "warnings": review_waivers.get("warnings", []), + }, + "review_annotations": { + "ok": review_annotations.get("ok", False), + "summary": review_annotations.get("summary", {}), + "annotations": review_annotations.get("annotations", [])[:8], + "failures": review_annotations.get("failures", []), + }, "synthesis_highlights": synthesis, "artifact_design": q_review["artifact_design"], "prompt_quality": q_review["prompt_quality"], diff --git a/scripts/trust_check.py b/scripts/trust_check.py index f0afeb4..5feb154 100644 --- a/scripts/trust_check.py +++ b/scripts/trust_check.py @@ -3,9 +3,13 @@ import argparse import ast import hashlib import json +import subprocess +import sys import re +from datetime import date from pathlib import Path from typing import Any +from urllib.parse import urlparse try: import yaml @@ -17,6 +21,13 @@ ROOT = Path(__file__).resolve().parent.parent SCAN_DIRS = ["agents", "docs", "evals", "references", "runtime", "scripts", "security", "skill-ir", "templates"] ROOT_FILES = ["SKILL.md", "README.md", "manifest.json", "requirements-ci.txt", "Makefile"] TEXT_SUFFIXES = {".md", ".json", ".jsonl", ".yaml", ".yml", ".py", ".sh", ".txt", ".toml"} +PACKAGE_HASH_SCOPE = "source-contract-without-generated-reports" +INTERNAL_SCRIPT_INTERFACE = "internal-module" +NETWORK_POLICY_REL_PATH = "security/network_policy.json" +PERMISSION_POLICY_REL_PATH = "security/permission_policy.json" +HELP_SMOKE_TIMEOUT_SECONDS = 5.0 +PERMISSION_CAPABILITIES = ("network", "file_write", "subprocess", "interactive") +PERMISSION_TARGETS = ("openai", "claude", "generic") SECRET_PATTERNS = [ ("private_key", re.compile(r"-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY-----")), ("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b")), @@ -91,19 +102,80 @@ def script_inventory(skill_dir: Path) -> list[dict[str, Any]]: for path in sorted(scripts_dir.glob("*.py")): text = path.read_text(encoding="utf-8", errors="replace") flags = script_flags(text) + interface = script_interface(text) + urls = extract_url_literals(text) inventory.append( { "path": relpath(skill_dir, path), + "interface": interface["name"], + "interface_declared": interface["declared"], + "interface_reason": interface["reason"], "has_argparse": "argparse" in text, "has_main_guard": 'if __name__ == "__main__"' in text, "uses_input": flags["uses_input"], "uses_network": flags["uses_network"], + "uses_file_write": flags["uses_file_write"], "uses_subprocess": flags["uses_subprocess"], + "network_urls": urls, + "network_hosts": sorted({urlparse(url).hostname or "" for url in urls if urlparse(url).hostname}), } ) return inventory +def string_assignment(tree: ast.Module, variable_name: str) -> str: + for node in tree.body: + value_node = None + if isinstance(node, ast.Assign): + if any(isinstance(target, ast.Name) and target.id == variable_name for target in node.targets): + value_node = node.value + elif isinstance(node, ast.AnnAssign): + target = node.target + if isinstance(target, ast.Name) and target.id == variable_name: + value_node = node.value + if isinstance(value_node, ast.Constant) and isinstance(value_node.value, str): + return value_node.value + return "" + + +def script_interface(text: str) -> dict[str, Any]: + try: + tree = ast.parse(text) + except SyntaxError: + match = re.search(r"SCRIPT_INTERFACE\s*=\s*['\"]([^'\"]+)['\"]", text) + reason_match = re.search(r"SCRIPT_INTERFACE_REASON\s*=\s*['\"]([^'\"]+)['\"]", text) + name = match.group(1) if match else "cli" + reason = reason_match.group(1) if reason_match else "" + return {"name": name, "declared": bool(match), "reason": reason} + + name = string_assignment(tree, "SCRIPT_INTERFACE") + reason = string_assignment(tree, "SCRIPT_INTERFACE_REASON") + if name: + return {"name": name, "declared": True, "reason": reason} + return {"name": "cli", "declared": False, "reason": "Default CLI classification; add SCRIPT_INTERFACE for internal modules."} + + +def extract_url_literals(text: str) -> list[str]: + values: list[str] = [] + try: + tree = ast.parse(text) + for node in ast.walk(tree): + if isinstance(node, ast.Constant) and isinstance(node.value, str): + values.append(node.value) + except SyntaxError: + values = re.findall(r"['\"]([^'\"]+)['\"]", text) + + urls = [] + seen = set() + for value in values: + for match in re.finditer(r"https?://[^\s'\"<>]+", value): + url = match.group(0).rstrip(").,]") + if url not in seen: + seen.add(url) + urls.append(url) + return urls + + def call_name(node: ast.AST) -> str: if isinstance(node, ast.Name): return node.id @@ -120,16 +192,39 @@ def script_flags(text: str) -> dict[str, bool]: return { "uses_input": bool(re.search(r"\b(?:input|getpass)\s*\(", text)), "uses_network": bool(re.search(r"\b(?:urlopen|Request)\s*\(|\brequests\.", text)), + "uses_file_write": bool( + re.search(r"\.(?:write_text|write_bytes|mkdir|unlink)\s*\(", text) + or re.search(r"\b(?:open)\s*\([^)]*['\"][wa+x]", text) + or "shutil.rmtree" in text + ), "uses_subprocess": "subprocess." in text, } - calls = [call_name(node.func) for node in ast.walk(tree) if isinstance(node, ast.Call)] + call_nodes = [node for node in ast.walk(tree) if isinstance(node, ast.Call)] + calls = [call_name(node.func) for node in call_nodes] return { "uses_input": any(name in {"input", "getpass.getpass"} or name.endswith(".getpass") for name in calls), "uses_network": any(name in {"urlopen", "Request"} or name.startswith("requests.") for name in calls), + "uses_file_write": any(is_file_write_call(node, call_name(node.func)) for node in call_nodes), "uses_subprocess": any(name.startswith("subprocess.") for name in calls), } +def is_file_write_call(node: ast.Call, name: str) -> bool: + if name.endswith((".write_text", ".write_bytes", ".mkdir", ".unlink", ".rmdir")): + return True + if name in {"shutil.copy", "shutil.copy2", "shutil.copytree", "shutil.move", "shutil.rmtree"}: + return True + if name == "zipfile.ZipFile": + return True + if name in {"open", "Path.open"} or name.endswith(".open"): + args = list(node.args) + keywords = {keyword.arg: keyword.value for keyword in node.keywords if keyword.arg} + mode_node = args[1] if len(args) > 1 else keywords.get("mode") + if isinstance(mode_node, ast.Constant) and isinstance(mode_node.value, str): + return any(flag in mode_node.value for flag in ("w", "a", "x", "+")) + return False + + def dependency_status(skill_dir: Path) -> dict[str, Any]: candidates = ["requirements-ci.txt", "requirements.txt", "pyproject.toml", "package-lock.json", "uv.lock", "poetry.lock"] present = [name for name in candidates if (skill_dir / name).exists()] @@ -148,6 +243,179 @@ def dependency_status(skill_dir: Path) -> dict[str, Any]: return {"present": present, "pinned": pinned, "unpinned": unpinned} +def script_capability_paths(scripts: list[dict[str, Any]]) -> dict[str, list[str]]: + return { + "network": [item["path"] for item in scripts if item["uses_network"]], + "file_write": [item["path"] for item in scripts if item.get("uses_file_write", False)], + "subprocess": [item["path"] for item in scripts if item["uses_subprocess"]], + "interactive": [item["path"] for item in scripts if item["uses_input"]], + } + + +def load_network_policy(skill_dir: Path) -> dict[str, Any]: + path = skill_dir / NETWORK_POLICY_REL_PATH + if not path.exists(): + return {"present": False, "path": NETWORK_POLICY_REL_PATH, "scripts": {}} + payload = load_json(path) + scripts = payload.get("scripts", {}) + return { + "present": True, + "path": NETWORK_POLICY_REL_PATH, + "schema_version": payload.get("schema_version", ""), + "default_policy": payload.get("default_policy", {}), + "scripts": scripts if isinstance(scripts, dict) else {}, + } + + +def load_permission_policy(skill_dir: Path) -> dict[str, Any]: + path = skill_dir / PERMISSION_POLICY_REL_PATH + if not path.exists(): + return {"present": False, "path": PERMISSION_POLICY_REL_PATH, "capabilities": {}} + payload = load_json(path) + capabilities = payload.get("capabilities", {}) + return { + "present": True, + "path": PERMISSION_POLICY_REL_PATH, + "schema_version": payload.get("schema_version", ""), + "reviewed_at": payload.get("reviewed_at", ""), + "capabilities": capabilities if isinstance(capabilities, dict) else {}, + } + + +def parse_iso_date(value: Any) -> date | None: + try: + return date.fromisoformat(str(value)[:10]) + except (TypeError, ValueError): + return None + + +def permission_governance_status(skill_dir: Path, scripts: list[dict[str, Any]], today: date | None = None) -> dict[str, Any]: + policy = load_permission_policy(skill_dir) + today = today or date.today() + capability_paths = script_capability_paths(scripts) + required = {name: paths for name, paths in capability_paths.items() if paths} + approvals = [] + missing = [] + invalid = [] + expired = [] + approved = [] + + for name, paths in required.items(): + entry = policy["capabilities"].get(name, {}) if policy["present"] else {} + if not isinstance(entry, dict) or not entry: + missing.append(name) + approvals.append({"capability": name, "status": "missing", "scripts": paths, "validation": ["approval entry is missing"]}) + continue + validation = [] + if entry.get("decision") != "approved": + validation.append("decision must be approved") + if not entry.get("reviewer"): + validation.append("reviewer is required") + if len(str(entry.get("reason", ""))) < 30: + validation.append("reason must be at least 30 characters") + if not entry.get("scope"): + validation.append("scope is required") + expires_at = parse_iso_date(entry.get("expires_at")) + if expires_at is None: + validation.append("expires_at must be ISO date") + target_enforcement = entry.get("target_enforcement", {}) + if not isinstance(target_enforcement, dict): + validation.append("target_enforcement must be an object") + target_enforcement = {} + missing_targets = [target for target in PERMISSION_TARGETS if not target_enforcement.get(target)] + if missing_targets: + validation.append(f"target_enforcement missing: {', '.join(missing_targets)}") + if validation: + invalid.append(name) + status = "invalid" + elif expires_at is not None and expires_at < today: + expired.append(name) + status = "expired" + else: + approved.append(name) + status = "approved" + approvals.append( + { + "capability": name, + "status": status, + "scripts": paths, + "reviewer": str(entry.get("reviewer", "")), + "scope": str(entry.get("scope", "")), + "reason": str(entry.get("reason", "")), + "expires_at": str(entry.get("expires_at", "")), + "evidence": entry.get("evidence", []) if isinstance(entry.get("evidence", []), list) else [], + "target_enforcement": target_enforcement, + "validation": validation, + } + ) + + return { + "present": policy["present"], + "path": policy["path"], + "schema_version": policy.get("schema_version", ""), + "reviewed_at": policy.get("reviewed_at", ""), + "required_capabilities": sorted(required), + "approved_capabilities": sorted(approved), + "missing_capabilities": sorted(missing), + "invalid_capabilities": sorted(invalid), + "expired_capabilities": sorted(expired), + "approval_count": len(approved), + "required_count": len(required), + "missing_count": len(missing), + "invalid_count": len(invalid), + "expired_count": len(expired), + "approvals": approvals, + } + + +def network_policy_status(skill_dir: Path, scripts: list[dict[str, Any]]) -> dict[str, Any]: + policy = load_network_policy(skill_dir) + network_scripts = [item for item in scripts if item["uses_network"]] + covered = [] + missing = [] + mismatches = [] + for item in network_scripts: + entry = policy["scripts"].get(item["path"], {}) if policy["present"] else {} + if not isinstance(entry, dict) or not entry: + missing.append(item["path"]) + continue + allowed_hosts = set(entry.get("allowed_hosts") or []) + if not allowed_hosts: + mismatches.append( + { + "path": item["path"], + "reason": "allowed_hosts is empty", + "observed_hosts": item["network_hosts"], + "allowed_hosts": [], + } + ) + continue + observed_hosts = set(item.get("network_hosts") or []) + unexpected_hosts = sorted(observed_hosts - allowed_hosts) + if unexpected_hosts: + mismatches.append( + { + "path": item["path"], + "reason": "observed HTTPS hosts are not declared in policy", + "observed_hosts": sorted(observed_hosts), + "allowed_hosts": sorted(allowed_hosts), + "unexpected_hosts": unexpected_hosts, + } + ) + continue + covered.append(item["path"]) + return { + "present": policy["present"], + "path": policy["path"], + "schema_version": policy.get("schema_version", ""), + "network_script_count": len(network_scripts), + "covered_scripts": covered, + "missing_scripts": missing, + "mismatches": mismatches, + "default_policy": policy.get("default_policy", {}), + } + + def interface_trust(skill_dir: Path) -> dict[str, Any]: interface = load_yaml(skill_dir / "agents" / "interface.yaml") trust = interface.get("compatibility", {}).get("trust", {}) @@ -169,12 +437,106 @@ def package_digest(files: list[Path], skill_dir: Path) -> str: return digest.hexdigest() -def build_trust_report(skill_dir: Path) -> dict[str, Any]: +def help_smoke_candidates(scripts: list[dict[str, Any]]) -> list[dict[str, Any]]: + return [ + item + for item in scripts + if item["interface"] != INTERNAL_SCRIPT_INTERFACE and item["has_argparse"] + ] + + +def run_help_smoke_checks(skill_dir: Path, scripts: list[dict[str, Any]], timeout: float) -> dict[str, Any]: + candidates = help_smoke_candidates(scripts) + skipped = [ + { + "path": item["path"], + "reason": "internal module" if item["interface"] == INTERNAL_SCRIPT_INTERFACE else "missing argparse/help surface", + } + for item in scripts + if item not in candidates + ] + results = [] + for item in candidates: + command = [sys.executable, item["path"], "--help"] + try: + proc = subprocess.run( + command, + cwd=skill_dir, + capture_output=True, + text=True, + timeout=timeout, + ) + output = f"{proc.stdout}\n{proc.stderr}".lower() + has_help_text = "usage:" in output or "--help" in output + results.append( + { + "path": item["path"], + "command": f"python3 {item['path']} --help", + "returncode": proc.returncode, + "timed_out": False, + "passed": proc.returncode == 0 and has_help_text, + "has_help_text": has_help_text, + "stdout_excerpt": proc.stdout.strip()[:240], + "stderr_excerpt": proc.stderr.strip()[:240], + } + ) + except subprocess.TimeoutExpired as exc: + results.append( + { + "path": item["path"], + "command": f"python3 {item['path']} --help", + "returncode": None, + "timed_out": True, + "passed": False, + "has_help_text": False, + "stdout_excerpt": (exc.stdout or "").strip()[:240] if isinstance(exc.stdout, str) else "", + "stderr_excerpt": (exc.stderr or "").strip()[:240] if isinstance(exc.stderr, str) else "", + } + ) + failed = [item for item in results if not item["passed"]] + return { + "enabled": True, + "timeout_seconds": timeout, + "candidate_count": len(candidates), + "checked_count": len(results), + "passed_count": len(results) - len(failed), + "failed_count": len(failed), + "skipped_count": len(skipped), + "failed_scripts": [item["path"] for item in failed], + "results": results, + "skipped": skipped, + } + + +def disabled_help_smoke_status(scripts: list[dict[str, Any]], timeout: float) -> dict[str, Any]: + candidates = help_smoke_candidates(scripts) + return { + "enabled": False, + "timeout_seconds": timeout, + "candidate_count": len(candidates), + "checked_count": 0, + "passed_count": 0, + "failed_count": 0, + "skipped_count": len(scripts), + "failed_scripts": [], + "results": [], + "skipped": [{"path": item["path"], "reason": "help smoke disabled"} for item in scripts], + } + + +def build_trust_report(skill_dir: Path, run_help_smoke: bool = True, help_smoke_timeout: float = HELP_SMOKE_TIMEOUT_SECONDS) -> dict[str, Any]: skill_dir = skill_dir.resolve() files = iter_scan_files(skill_dir) secrets = scan_secrets(skill_dir, files) scripts = script_inventory(skill_dir) deps = dependency_status(skill_dir) + network_policy = network_policy_status(skill_dir, scripts) + help_smoke = ( + run_help_smoke_checks(skill_dir, scripts, help_smoke_timeout) + if run_help_smoke + else disabled_help_smoke_status(scripts, help_smoke_timeout) + ) + permission_governance = permission_governance_status(skill_dir, scripts) trust = interface_trust(skill_dir) failures = [] warnings = [] @@ -187,23 +549,51 @@ def build_trust_report(skill_dir: Path) -> dict[str, Any]: warnings.append(f"Unpinned dependency entries: {', '.join(deps['unpinned'])}") if not deps["present"]: warnings.append("No dependency or lock file detected") - missing_help = [item["path"] for item in scripts if not item["has_argparse"]] + internal_modules = [item for item in scripts if item["interface"] == INTERNAL_SCRIPT_INTERFACE] + missing_help = [ + item["path"] for item in scripts if item["interface"] != INTERNAL_SCRIPT_INTERFACE and not item["has_argparse"] + ] if missing_help: - warnings.append(f"Scripts without argparse/help surface: {', '.join(missing_help[:8])}") + warnings.append(f"CLI scripts without argparse/help surface: {', '.join(missing_help[:8])}") interactive = [item["path"] for item in scripts if item["uses_input"]] if interactive: warnings.append(f"Interactive scripts require reviewer awareness: {', '.join(interactive[:8])}") network = [item["path"] for item in scripts if item["uses_network"]] - if network: - warnings.append(f"Network-capable scripts require bounded host policy: {', '.join(network[:8])}") + file_write = [item["path"] for item in scripts if item["uses_file_write"]] + if network_policy["missing_scripts"]: + warnings.append(f"Network-capable scripts require bounded host policy: {', '.join(network_policy['missing_scripts'][:8])}") + if network_policy["mismatches"]: + warning_paths = [item["path"] for item in network_policy["mismatches"]] + warnings.append(f"Network host policy mismatch: {', '.join(warning_paths[:8])}") + if help_smoke["failed_scripts"]: + warnings.append(f"CLI help smoke failed: {', '.join(help_smoke['failed_scripts'][:8])}") + if permission_governance["missing_capabilities"]: + warnings.append(f"Permission approvals missing: {', '.join(permission_governance['missing_capabilities'][:8])}") + if permission_governance["invalid_capabilities"]: + warnings.append(f"Permission approvals invalid: {', '.join(permission_governance['invalid_capabilities'][:8])}") + if permission_governance["expired_capabilities"]: + warnings.append(f"Permission approvals expired: {', '.join(permission_governance['expired_capabilities'][:8])}") summary = { "scanned_files": len(files), "script_count": len(scripts), + "internal_module_count": len(internal_modules), "secret_findings": len(secrets), "dependency_files": deps["present"], "network_script_count": len(network), + "network_policy_covered_count": len(network_policy["covered_scripts"]), + "network_policy_missing_count": len(network_policy["missing_scripts"]), + "file_write_script_count": len(file_write), + "permission_required_count": permission_governance["required_count"], + "permission_approved_count": permission_governance["approval_count"], + "permission_missing_count": permission_governance["missing_count"], + "permission_invalid_count": permission_governance["invalid_count"], + "permission_expired_count": permission_governance["expired_count"], + "help_smoke_checked_count": help_smoke["checked_count"], + "help_smoke_failed_count": help_smoke["failed_count"], "interactive_script_count": len(interactive), + "package_hash_scope": PACKAGE_HASH_SCOPE, + "package_hash_file_count": len(files), "package_sha256": package_digest(files, skill_dir), } return { @@ -215,6 +605,9 @@ def build_trust_report(skill_dir: Path) -> dict[str, Any]: "secrets": secrets, "scripts": scripts, "dependencies": deps, + "network_policy": network_policy, + "help_smoke": help_smoke, + "permission_governance": permission_governance, "trust_metadata": trust, } @@ -227,9 +620,19 @@ def render_markdown(payload: dict[str, Any]) -> str: f"- OK: `{payload['ok']}`", f"- Scanned files: `{summary['scanned_files']}`", f"- Scripts: `{summary['script_count']}`", + f"- Internal script modules: `{summary.get('internal_module_count', 0)}`", f"- Secret findings: `{summary['secret_findings']}`", f"- Network-capable scripts: `{summary['network_script_count']}`", + f"- Network policy covered scripts: `{summary.get('network_policy_covered_count', 0)}`", + f"- Network policy missing scripts: `{summary.get('network_policy_missing_count', 0)}`", + f"- File-write scripts: `{summary.get('file_write_script_count', 0)}`", + f"- Permission approvals: `{summary.get('permission_approved_count', 0)} / {summary.get('permission_required_count', 0)}`", + f"- Permission approval gaps: `{summary.get('permission_missing_count', 0) + summary.get('permission_invalid_count', 0) + summary.get('permission_expired_count', 0)}`", + f"- CLI help smoke checked: `{summary.get('help_smoke_checked_count', 0)}`", + f"- CLI help smoke failures: `{summary.get('help_smoke_failed_count', 0)}`", f"- Interactive scripts: `{summary['interactive_script_count']}`", + f"- Package hash scope: `{summary['package_hash_scope']}`", + f"- Package hash files: `{summary['package_hash_file_count']}`", f"- Package SHA256: `{summary['package_sha256']}`", "", "## Failures", @@ -242,10 +645,41 @@ def render_markdown(payload: dict[str, Any]) -> str: lines.append(f"- Files: `{', '.join(payload['dependencies']['present']) or 'none'}`") lines.append(f"- Pinned entries: `{len(payload['dependencies']['pinned'])}`") lines.append(f"- Unpinned entries: `{len(payload['dependencies']['unpinned'])}`") - lines.extend(["", "## Script Surface", "", "| Script | Argparse | Main Guard | Input | Network | Subprocess |", "| --- | --- | --- | --- | --- | --- |"]) + network_policy = payload.get("network_policy", {}) + lines.extend(["", "## Network Policy", ""]) + lines.append(f"- Policy file: `{network_policy.get('path', NETWORK_POLICY_REL_PATH)}`") + lines.append(f"- Present: `{network_policy.get('present', False)}`") + lines.append(f"- Covered scripts: `{len(network_policy.get('covered_scripts', []))}`") + lines.append(f"- Missing scripts: `{', '.join(network_policy.get('missing_scripts', [])) or 'none'}`") + lines.append(f"- Mismatches: `{len(network_policy.get('mismatches', []))}`") + permission_governance = payload.get("permission_governance", {}) + lines.extend(["", "## Permission Governance", ""]) + lines.append(f"- Policy file: `{permission_governance.get('path', PERMISSION_POLICY_REL_PATH)}`") + lines.append(f"- Present: `{permission_governance.get('present', False)}`") + lines.append(f"- Required capabilities: `{', '.join(permission_governance.get('required_capabilities', [])) or 'none'}`") + lines.append(f"- Approved capabilities: `{', '.join(permission_governance.get('approved_capabilities', [])) or 'none'}`") + lines.append(f"- Missing approvals: `{', '.join(permission_governance.get('missing_capabilities', [])) or 'none'}`") + lines.append(f"- Invalid approvals: `{', '.join(permission_governance.get('invalid_capabilities', [])) or 'none'}`") + lines.append(f"- Expired approvals: `{', '.join(permission_governance.get('expired_capabilities', [])) or 'none'}`") + help_smoke = payload.get("help_smoke", {}) + lines.extend(["", "## CLI Help Smoke", ""]) + lines.append(f"- Enabled: `{help_smoke.get('enabled', False)}`") + lines.append(f"- Timeout seconds: `{help_smoke.get('timeout_seconds', HELP_SMOKE_TIMEOUT_SECONDS)}`") + lines.append(f"- Checked scripts: `{help_smoke.get('checked_count', 0)}`") + lines.append(f"- Passed scripts: `{help_smoke.get('passed_count', 0)}`") + lines.append(f"- Failed scripts: `{', '.join(help_smoke.get('failed_scripts', [])) or 'none'}`") + lines.extend( + [ + "", + "## Script Surface", + "", + "| Script | Interface | Declared | Argparse | Main Guard | Input | Network | File Write | Subprocess | Reason |", + "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |", + ] + ) for item in payload["scripts"]: lines.append( - f"| {item['path']} | {item['has_argparse']} | {item['has_main_guard']} | {item['uses_input']} | {item['uses_network']} | {item['uses_subprocess']} |" + f"| {item['path']} | {item['interface']} | {item['interface_declared']} | {item['has_argparse']} | {item['has_main_guard']} | {item['uses_input']} | {item['uses_network']} | {item.get('uses_file_write', False)} | {item['uses_subprocess']} | {item['interface_reason']} |" ) return "\n".join(lines).strip() + "\n" @@ -255,12 +689,18 @@ def main() -> None: parser.add_argument("skill_dir", nargs="?", default=".") parser.add_argument("--output-json") parser.add_argument("--output-md") + parser.add_argument("--skip-help-smoke", action="store_true") + parser.add_argument("--help-smoke-timeout", type=float, default=HELP_SMOKE_TIMEOUT_SECONDS) args = parser.parse_args() skill_dir = Path(args.skill_dir).resolve() output_json = Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "security_trust_report.json" output_md = Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "security_trust_report.md" - payload = build_trust_report(skill_dir) + payload = build_trust_report( + skill_dir, + run_help_smoke=not args.skip_help_smoke, + help_smoke_timeout=args.help_smoke_timeout, + ) payload["artifacts"] = {"json": display_path(output_json), "markdown": display_path(output_md)} output_json.parent.mkdir(parents=True, exist_ok=True) output_md.parent.mkdir(parents=True, exist_ok=True) diff --git a/scripts/upgrade_check.py b/scripts/upgrade_check.py new file mode 100644 index 0000000..e801162 --- /dev/null +++ b/scripts/upgrade_check.py @@ -0,0 +1,268 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +BUMP_RANK = {"none": 0, "patch": 1, "minor": 2, "major": 3} +COMPAT_RANK = {"missing": 0, "unknown": 0, "fail": 0, "warn": 1, "pass": 2} + + +def display_path(path: Path, root: Path = ROOT) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def package_from_payload(payload: dict[str, Any]) -> dict[str, Any]: + package = payload.get("package") + if isinstance(package, dict): + return package + return payload + + +def parse_version(value: str) -> tuple[int, int, int] | None: + match = re.match(r"^(\d+)\.(\d+)\.(\d+)(?:[-+].*)?$", str(value or "")) + if not match: + return None + return tuple(int(part) for part in match.groups()) + + +def declared_bump(previous: str, current: str) -> str: + old = parse_version(previous) + new = parse_version(current) + if old is None or new is None: + return "invalid" + if new < old: + return "downgrade" + if new == old: + return "none" + if new[0] > old[0]: + return "major" + if new[1] > old[1]: + return "minor" + return "patch" + + +def compare_sets(previous: list[Any], current: list[Any]) -> tuple[list[str], list[str]]: + old = {str(item) for item in previous} + new = {str(item) for item in current} + return sorted(new - old), sorted(old - new) + + +def compatibility_changes(previous: dict[str, Any], current: dict[str, Any]) -> list[dict[str, str]]: + targets = sorted(set(previous) | set(current)) + changes = [] + for target in targets: + old = str(previous.get(target, "missing")) + new = str(current.get(target, "missing")) + if old != new: + direction = "regressed" if COMPAT_RANK.get(new, 0) < COMPAT_RANK.get(old, 0) else "improved" + changes.append({"target": target, "from": old, "to": new, "direction": direction}) + return changes + + +def metadata_changes(previous: dict[str, Any], current: dict[str, Any]) -> list[dict[str, str]]: + fields = ["description", "maturity", "owner", "review_cadence", "trust_level", "license"] + changes = [] + for field in fields: + old = str(previous.get(field, "")) + new = str(current.get(field, "")) + if old != new: + changes.append({"field": field, "from": old, "to": new}) + return changes + + +def checksum_changes(previous: dict[str, Any], current: dict[str, Any]) -> list[dict[str, str]]: + old_checksums = previous.get("checksums", {}) if isinstance(previous.get("checksums"), dict) else {} + new_checksums = current.get("checksums", {}) if isinstance(current.get("checksums"), dict) else {} + changes = [] + for key in sorted(set(old_checksums) | set(new_checksums)): + old = str(old_checksums.get(key, "")) + new = str(new_checksums.get(key, "")) + if old != new: + changes.append({"field": key, "from": old, "to": new}) + return changes + + +def required_bump( + *, + name_changed: bool, + removed_targets: list[str], + added_targets: list[str], + compat: list[dict[str, str]], + metadata: list[dict[str, str]], + checksums: list[dict[str, str]], +) -> str: + if name_changed or removed_targets or any(item["direction"] == "regressed" for item in compat): + return "major" + if added_targets or any(item["field"] in {"description", "maturity", "trust_level"} for item in metadata): + return "minor" + if metadata or checksums or compat: + return "patch" + return "none" + + +def build_release_notes(diff: dict[str, Any], recommended: str, current: dict[str, Any]) -> list[str]: + notes = [f"Recommended version bump: {recommended}."] + if diff["removed_targets"]: + notes.append(f"Breaking target removals: {', '.join(diff['removed_targets'])}.") + if diff["added_targets"]: + notes.append(f"Added targets: {', '.join(diff['added_targets'])}.") + regressions = [item for item in diff["compatibility_changes"] if item["direction"] == "regressed"] + if regressions: + notes.append("Compatibility regressions: " + ", ".join(f"{item['target']} {item['from']}->{item['to']}" for item in regressions) + ".") + if diff["checksum_changes"]: + notes.append("Package or archive checksum changed; reviewers should verify package artifacts before release.") + if current.get("artifacts", {}).get("package_verification"): + notes.append(f"Package verification evidence: {current['artifacts']['package_verification']}.") + return notes + + +def run_upgrade_check(previous: dict[str, Any], current: dict[str, Any], generated_at: str) -> dict[str, Any]: + failures = [] + warnings = [] + previous_version = str(previous.get("version", "")) + current_version = str(current.get("version", "")) + bump = declared_bump(previous_version, current_version) + if bump == "invalid": + failures.append("Version comparison requires semver-like previous and current versions") + if bump == "downgrade": + failures.append("Current version is lower than previous version") + + added_targets, removed_targets = compare_sets(previous.get("targets", []), current.get("targets", [])) + compat = compatibility_changes(previous.get("compatibility", {}), current.get("compatibility", {})) + metadata = metadata_changes(previous, current) + checksums = checksum_changes(previous, current) + name_changed = str(previous.get("name", "")) != str(current.get("name", "")) + recommended = required_bump( + name_changed=name_changed, + removed_targets=removed_targets, + added_targets=added_targets, + compat=compat, + metadata=metadata, + checksums=checksums, + ) + if bump in BUMP_RANK and BUMP_RANK[bump] < BUMP_RANK[recommended]: + failures.append(f"Version bump is insufficient: declared {bump}, recommended {recommended}") + if name_changed: + failures.append("Package name changed; publish as a new package or use a major version bump") + if not checksums: + warnings.append("No checksum changes detected; confirm the previous package is the intended comparison baseline") + + diff = { + "name_changed": name_changed, + "added_targets": added_targets, + "removed_targets": removed_targets, + "compatibility_changes": compat, + "metadata_changes": metadata, + "checksum_changes": checksums, + } + return { + "ok": not failures, + "schema_version": "2.0", + "generated_at": generated_at, + "previous": { + "name": previous.get("name", ""), + "version": previous_version, + "targets": previous.get("targets", []), + }, + "current": { + "name": current.get("name", ""), + "version": current_version, + "targets": current.get("targets", []), + }, + "summary": { + "declared_bump": bump, + "recommended_bump": recommended, + "breaking_change_count": int(name_changed) + len(removed_targets) + sum(1 for item in compat if item["direction"] == "regressed"), + "added_target_count": len(added_targets), + "metadata_change_count": len(metadata), + "checksum_change_count": len(checksums), + "failure_count": len(failures), + "warning_count": len(warnings), + }, + "upgrade_diff": diff, + "release_notes": build_release_notes(diff, recommended, current), + "failures": failures, + "warnings": warnings, + } + + +def render_markdown(report: dict[str, Any]) -> str: + lines = [ + "# Upgrade Check", + "", + f"- OK: `{report['ok']}`", + f"- Previous: `{report['previous']['name']} {report['previous']['version']}`", + f"- Current: `{report['current']['name']} {report['current']['version']}`", + f"- Declared bump: `{report['summary']['declared_bump']}`", + f"- Recommended bump: `{report['summary']['recommended_bump']}`", + f"- Breaking changes: `{report['summary']['breaking_change_count']}`", + "", + "## Upgrade Diff", + "", + f"- Added targets: `{', '.join(report['upgrade_diff']['added_targets']) or 'none'}`", + f"- Removed targets: `{', '.join(report['upgrade_diff']['removed_targets']) or 'none'}`", + f"- Compatibility changes: `{len(report['upgrade_diff']['compatibility_changes'])}`", + f"- Metadata changes: `{len(report['upgrade_diff']['metadata_changes'])}`", + f"- Checksum changes: `{len(report['upgrade_diff']['checksum_changes'])}`", + "", + "## Release Notes", + "", + ] + lines.extend(f"- {item}" for item in report["release_notes"]) + lines.extend(["", "## Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) + lines.extend(["", "## Warnings", ""]) + lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"]) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Compare current and previous registry package metadata for upgrade readiness.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--previous-package-json", required=True) + parser.add_argument("--current-package-json", default="reports/registry_audit.json") + parser.add_argument("--output-json", default="reports/upgrade_check.json") + parser.add_argument("--output-md", default="reports/upgrade_check.md") + parser.add_argument("--generated-at", default=str(date.today())) + args = parser.parse_args() + + previous = package_from_payload(load_json(Path(args.previous_package_json))) + current = package_from_payload(load_json(Path(args.current_package_json))) + report = run_upgrade_check(previous, current, args.generated_at) + report["artifacts"] = { + "previous_package": display_path(Path(args.previous_package_json)), + "current_package": display_path(Path(args.current_package_json)), + } + output_json = Path(args.output_json) + output_md = Path(args.output_md) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + report["artifacts"]["json"] = display_path(output_json) + report["artifacts"]["markdown"] = display_path(output_md) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/verify_package.py b/scripts/verify_package.py new file mode 100644 index 0000000..4b27727 --- /dev/null +++ b/scripts/verify_package.py @@ -0,0 +1,261 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import zipfile +from datetime import date +from pathlib import Path, PurePosixPath +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent + + +def display_path(path: Path, root: Path = ROOT) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def zip_names(zip_path: Path) -> list[str]: + with zipfile.ZipFile(zip_path) as archive: + return archive.namelist() + + +def unsafe_zip_entries(names: list[str]) -> list[str]: + unsafe = [] + for name in names: + pure = PurePosixPath(name) + if pure.is_absolute() or ".." in pure.parts or name.startswith("\\") or (pure.parts and ":" in pure.parts[0]): + unsafe.append(name) + return unsafe + + +def generated_zip_entries(names: list[str]) -> list[str]: + generated = [] + for name in names: + parts = PurePosixPath(name).parts + if "dist" in parts or (len(parts) > 2 and parts[1] == "tests" and any(part.startswith("tmp") for part in parts[2:])): + generated.append(name) + return generated + + +def required_targets(expectations: dict[str, Any], package_dir: Path) -> list[str]: + targets = expectations.get("required_targets") or [] + if targets: + return [str(item) for item in targets] + targets_dir = package_dir / "targets" + if not targets_dir.exists(): + return [] + return sorted(path.name for path in targets_dir.iterdir() if path.is_dir()) + + +def add_check(checks: list[dict[str, str]], failures: list[str], check_id: str, passed: bool, detail: str) -> None: + checks.append({"id": check_id, "status": "pass" if passed else "fail", "detail": detail}) + if not passed: + failures.append(detail) + + +def verify_package( + skill_dir: Path, + package_dir: Path, + expectations: dict[str, Any], + registry: dict[str, Any], + require_zip: bool, + generated_at: str, +) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + package_dir = package_dir.resolve() + checks: list[dict[str, str]] = [] + failures: list[str] = [] + warnings: list[str] = [] + + manifest_path = package_dir / "manifest.json" + manifest = load_json(manifest_path) + add_check(checks, failures, "package-manifest", bool(manifest), f"Package manifest exists: {display_path(manifest_path)}") + + targets = required_targets(expectations, package_dir) + adapter_paths = [] + for target in targets: + adapter_path = package_dir / "targets" / target / "adapter.json" + adapter_paths.append(adapter_path) + adapter = load_json(adapter_path) + add_check(checks, failures, f"{target}-adapter", bool(adapter), f"Adapter exists for target: {target}") + for field in expectations.get("required_fields", []): + add_check( + checks, + failures, + f"{target}-field-{field}", + field in adapter, + f"{target} adapter includes field: {field}", + ) + for target, key in ( + ("openai", "openai_required_files"), + ("claude", "claude_required_files"), + ("generic", "generic_required_files"), + ): + for rel in expectations.get(key, []): + add_check(checks, failures, f"{target}-file-{rel}", (package_dir / rel).exists(), f"Package contains {rel}") + + archive_path = package_dir / f"{skill_dir.name}.zip" + archive_sha = "" + archive_entries: list[str] = [] + if archive_path.exists(): + archive_sha = sha256_file(archive_path) + try: + archive_entries = zip_names(archive_path) + except zipfile.BadZipFile: + add_check(checks, failures, "archive-readable", False, f"Archive is not a readable zip: {display_path(archive_path)}") + else: + unsafe_entries = unsafe_zip_entries(archive_entries) + required_entries = [ + f"{skill_dir.name}/SKILL.md", + f"{skill_dir.name}/manifest.json", + f"{skill_dir.name}/agents/interface.yaml", + ] + add_check(checks, failures, "archive-safe-paths", not unsafe_entries, "Archive has no absolute or parent-traversal entries") + for entry in required_entries: + add_check(checks, failures, f"archive-entry-{entry}", entry in archive_entries, f"Archive contains {entry}") + generated_entries = generated_zip_entries(archive_entries) + add_check(checks, failures, "archive-excludes-generated", not generated_entries, "Archive excludes generated dist/ and tests/tmp* contents") + elif require_zip: + add_check(checks, failures, "archive-present", False, f"Missing required package archive: {display_path(archive_path)}") + else: + warnings.append(f"Package archive not found: {display_path(archive_path)}") + + registry_package = registry.get("package", {}) if registry else {} + if registry_package: + add_check(checks, failures, "registry-ok", bool(registry.get("ok")), "Registry audit is OK") + add_check( + checks, + failures, + "registry-name-match", + registry_package.get("name") == manifest.get("name"), + "Registry package name matches package manifest", + ) + add_check( + checks, + failures, + "registry-version-match", + registry_package.get("version") == manifest.get("version"), + "Registry package version matches package manifest", + ) + compatibility = registry_package.get("compatibility", {}) + for target in targets: + add_check( + checks, + failures, + f"registry-compat-{target}", + compatibility.get(target) in {"pass", "warn"}, + f"Registry compatibility is reviewable for target: {target}", + ) + else: + warnings.append("Registry audit was not supplied; package verification skipped metadata parity checks.") + + report = { + "ok": not failures, + "schema_version": "2.0", + "generated_at": generated_at, + "skill_dir": display_path(skill_dir), + "package_dir": display_path(package_dir), + "summary": { + "target_count": len(targets), + "adapter_count": sum(1 for path in adapter_paths if path.exists()), + "archive_present": archive_path.exists(), + "archive_sha256": archive_sha, + "archive_entry_count": len(archive_entries), + "failure_count": len(failures), + "warning_count": len(warnings), + }, + "checks": checks, + "failures": failures, + "warnings": warnings, + "artifacts": { + "manifest": display_path(manifest_path), + "archive": display_path(archive_path) if archive_path.exists() else "", + }, + } + return report + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Package Verification", + "", + f"- OK: `{report['ok']}`", + f"- Package directory: `{report['package_dir']}`", + f"- Targets: `{summary['adapter_count']} / {summary['target_count']}` adapters present", + f"- Archive present: `{summary['archive_present']}`", + f"- Archive SHA256: `{summary['archive_sha256'] or 'n/a'}`", + f"- Failures: `{summary['failure_count']}`", + f"- Warnings: `{summary['warning_count']}`", + "", + "## Checks", + "", + "| Check | Status | Detail |", + "| --- | --- | --- |", + ] + for item in report["checks"]: + lines.append(f"| `{item['id']}` | `{item['status']}` | {item['detail']} |") + lines.extend(["", "## Failures", ""]) + lines.extend([f"- {item}" for item in report["failures"]] or ["- None"]) + lines.extend(["", "## Warnings", ""]) + lines.extend([f"- {item}" for item in report["warnings"]] or ["- None"]) + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Verify generated skill package artifacts and archive integrity.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--package-dir", default="dist") + parser.add_argument("--expectations", default="evals/packaging_expectations.json") + parser.add_argument("--registry-json", default="reports/registry_audit.json") + parser.add_argument("--output-json", default="reports/package_verification.json") + parser.add_argument("--output-md", default="reports/package_verification.md") + parser.add_argument("--require-zip", action="store_true") + parser.add_argument("--generated-at", default=str(date.today())) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir) + package_dir = Path(args.package_dir) + if not package_dir.is_absolute(): + package_dir = Path.cwd() / package_dir + expectations = load_json(Path(args.expectations)) if args.expectations else {} + registry = load_json(Path(args.registry_json)) if args.registry_json else {} + report = verify_package(skill_dir, package_dir, expectations, registry, args.require_zip, args.generated_at) + + output_json = Path(args.output_json) + output_md = Path(args.output_md) + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + report["artifacts"]["json"] = display_path(output_json) + report["artifacts"]["markdown"] = display_path(output_md) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(0 if report["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/yao.py b/scripts/yao.py index 61bc7ba..9fe8604 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -62,6 +62,10 @@ def script_path(name: str) -> str: return str(SCRIPTS / name) +def local_output_runner_command() -> str: + return json.dumps(["python3", "scripts/local_output_eval_runner.py"]) + + def load_json_maybe(text: str) -> dict | None: text = text.strip() if not text: @@ -660,6 +664,13 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_artifact_design_profile.py", [str(ROOT)]), run_script("render_prompt_quality_profile.py", [str(ROOT)]), run_script("render_system_model.py", [str(ROOT)]), + run_script("compile_skill.py", [str(ROOT)]), + run_script("run_output_eval.py", []), + run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()]), + run_script("adjudicate_output_review.py", []), + run_script("render_adoption_drift_report.py", [str(ROOT)]), + run_script("render_review_waivers.py", [str(ROOT)]), + run_script("render_review_annotations.py", [str(ROOT)]), ] ) report = { @@ -679,6 +690,12 @@ def command_report(args: argparse.Namespace) -> int: "artifact_design_profile": "reports/artifact-design-profile.json", "prompt_quality_profile": "reports/prompt-quality-profile.json", "system_model": "reports/system-model.json", + "compiled_targets": "reports/compiled_targets.json", + "output_execution": "reports/output_execution_runs.json", + "output_review_adjudication": "reports/output_review_adjudication.json", + "adoption_drift": "reports/adoption_drift_report.json", + "review_waivers": "reports/review_waivers.json", + "review_annotations": "reports/review_annotations.json", }, } print(json.dumps(report, ensure_ascii=False, indent=2)) @@ -866,6 +883,112 @@ def command_feedback(args: argparse.Namespace) -> int: return 0 if report["ok"] else 2 +def command_adoption_drift(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.events_jsonl: + cmd.extend(["--events-jsonl", args.events_jsonl]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + if args.record_event: + cmd.extend(["--record-event", args.record_event]) + cmd.extend(["--activation-type", args.activation_type]) + cmd.extend(["--outcome", args.outcome]) + cmd.extend(["--failure-type", args.failure_type]) + if args.timestamp: + cmd.extend(["--timestamp", args.timestamp]) + if args.skill_name: + cmd.extend(["--skill-name", args.skill_name]) + if args.version: + cmd.extend(["--version", args.version]) + result = run_script("render_adoption_drift_report.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_review_waivers(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.waivers_json: + cmd.extend(["--waivers-json", args.waivers_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + if args.add_waiver: + required = { + "--gate-key": args.gate_key, + "--reviewer": args.reviewer, + "--reason": args.reason, + "--expires-at": args.expires_at, + } + missing = [name for name, value in required.items() if not value] + if missing: + print(json.dumps({"ok": False, "failures": [f"Missing required fields for --add-waiver: {', '.join(missing)}"]}, ensure_ascii=False, indent=2)) + return 2 + cmd.append("--add-waiver") + cmd.extend(["--gate-key", args.gate_key]) + cmd.extend(["--decision", args.decision]) + cmd.extend(["--reviewer", args.reviewer]) + cmd.extend(["--reason", args.reason]) + cmd.extend(["--expires-at", args.expires_at]) + if args.created_at: + cmd.extend(["--created-at", args.created_at]) + if args.evidence: + cmd.extend(["--evidence", args.evidence]) + if args.scope: + cmd.extend(["--scope", args.scope]) + result = run_script("render_review_waivers.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_review_annotations(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.annotations_json: + cmd.extend(["--annotations-json", args.annotations_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.write_template: + cmd.append("--write-template") + if args.add_annotation: + cmd.append("--add-annotation") + if args.annotation_id: + cmd.extend(["--annotation-id", args.annotation_id]) + if args.gate_key: + cmd.extend(["--gate-key", args.gate_key]) + if args.target_path: + cmd.extend(["--target-path", args.target_path]) + if args.line is not None: + cmd.extend(["--line", str(args.line)]) + if args.severity: + cmd.extend(["--severity", args.severity]) + if args.status: + cmd.extend(["--status", args.status]) + if args.reviewer: + cmd.extend(["--reviewer", args.reviewer]) + if args.created_at: + cmd.extend(["--created-at", args.created_at]) + if args.body: + cmd.extend(["--body", args.body]) + if args.suggested_action: + cmd.extend(["--suggested-action", args.suggested_action]) + if args.evidence: + cmd.extend(["--evidence", args.evidence]) + result = run_script("render_review_annotations.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_baseline_compare(args: argparse.Namespace) -> int: result = run_script("render_baseline_compare.py", baseline_compare_args()) print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) @@ -883,6 +1006,21 @@ def command_skill_ir(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_compile_skill(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + for target in args.target or []: + cmd.extend(["--target", target]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("compile_skill.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_output_eval(args: argparse.Namespace) -> int: cmd = [] if args.cases: @@ -891,11 +1029,53 @@ def command_output_eval(args: argparse.Namespace) -> int: cmd.extend(["--output-json", args.output_json]) if args.output_md: cmd.extend(["--output-md", args.output_md]) + if args.blind_pack_json: + cmd.extend(["--blind-pack-json", args.blind_pack_json]) + if args.blind_pack_md: + cmd.extend(["--blind-pack-md", args.blind_pack_md]) + if args.blind_answer_key_json: + cmd.extend(["--blind-answer-key-json", args.blind_answer_key_json]) result = run_script("run_output_eval.py", cmd) print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) return 0 if result["ok"] else 2 +def command_output_execution(args: argparse.Namespace) -> int: + cmd = [] + if args.cases: + cmd.extend(["--cases", args.cases]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.runner_command: + cmd.extend(["--runner-command", args.runner_command]) + if args.timeout_seconds is not None: + cmd.extend(["--timeout-seconds", str(args.timeout_seconds)]) + result = run_script("run_output_execution.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_output_review(args: argparse.Namespace) -> int: + cmd = [] + if args.blind_pack: + cmd.extend(["--blind-pack", args.blind_pack]) + if args.answer_key: + cmd.extend(["--answer-key", args.answer_key]) + if args.decisions: + cmd.extend(["--decisions", args.decisions]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.write_template: + cmd.append("--write-template") + result = run_script("adjudicate_output_review.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_conformance(args: argparse.Namespace) -> int: cmd = [str(Path(args.skill_dir).resolve())] for target in args.target or []: @@ -909,6 +1089,21 @@ def command_conformance(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_runtime_permissions(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + for target in args.target or []: + cmd.extend(["--target", target]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + result = run_script("probe_runtime_permissions.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_trust(args: argparse.Namespace) -> int: cmd = [str(Path(args.skill_dir).resolve())] if args.output_json: @@ -937,6 +1132,75 @@ def command_skill_atlas(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_registry_audit(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.registry_dir: + cmd.extend(["--registry-dir", args.registry_dir]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("registry_audit.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_package_verify(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + if args.expectations: + cmd.extend(["--expectations", args.expectations]) + if args.registry_json: + cmd.extend(["--registry-json", args.registry_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.require_zip: + cmd.append("--require-zip") + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("verify_package.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_install_simulate(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + if args.install_root: + cmd.extend(["--install-root", args.install_root]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("simulate_install.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_upgrade_check(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + cmd.extend(["--previous-package-json", args.previous_package_json]) + if args.current_package_json: + cmd.extend(["--current-package-json", args.current_package_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("upgrade_check.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_review(args: argparse.Namespace) -> int: target_name = resolve_promotion_target(args.target) bundle_dir = ROOT / "reports" / "iteration_bundles" / target_name @@ -997,6 +1261,10 @@ def command_workspace_flow(args: argparse.Namespace) -> int: {"phase": "report-refresh", "result": run_script("render_regression_history.py", [])}, {"phase": "report-refresh", "result": run_script("render_context_reports.py", [])}, {"phase": "report-refresh", "result": run_script("render_portability_report.py", [])}, + {"phase": "report-refresh", "result": run_script("compile_skill.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_adoption_drift_report.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_review_waivers.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_review_annotations.py", [str(ROOT)])}, ] ) @@ -1325,6 +1593,127 @@ def build_parser() -> argparse.ArgumentParser: feedback_cmd.add_argument("--recommended-action", default="review") feedback_cmd.set_defaults(func=command_feedback) + adoption_drift_cmd = subparsers.add_parser( + "adoption-drift", + help="Render local-first metadata-only adoption and drift telemetry for a skill package.", + ) + adoption_drift_cmd.add_argument("skill_dir", nargs="?", default=".") + adoption_drift_cmd.add_argument("--events-jsonl") + adoption_drift_cmd.add_argument("--output-json") + adoption_drift_cmd.add_argument("--output-md") + adoption_drift_cmd.add_argument("--generated-at") + adoption_drift_cmd.add_argument( + "--record-event", + choices=["review_event", "script_run", "skill_activation", "skill_output"], + ) + adoption_drift_cmd.add_argument( + "--activation-type", + choices=["explicit", "implicit", "manual", "unknown"], + default="unknown", + ) + adoption_drift_cmd.add_argument( + "--outcome", + choices=["accepted", "edited", "failed", "missed", "rejected", "reviewed", "unknown"], + default="unknown", + ) + adoption_drift_cmd.add_argument( + "--failure-type", + choices=[ + "bad_output", + "missing_resource", + "none", + "review_overdue", + "script_error", + "under_trigger", + "wrong_trigger", + ], + default="none", + ) + adoption_drift_cmd.add_argument("--timestamp") + adoption_drift_cmd.add_argument("--skill-name") + adoption_drift_cmd.add_argument("--version") + adoption_drift_cmd.set_defaults(func=command_adoption_drift) + + review_waivers_cmd = subparsers.add_parser( + "review-waivers", + help="Render or update human reviewer waiver evidence for Review Studio.", + ) + review_waivers_cmd.add_argument("skill_dir", nargs="?", default=".") + review_waivers_cmd.add_argument("--waivers-json") + review_waivers_cmd.add_argument("--output-json") + review_waivers_cmd.add_argument("--output-md") + review_waivers_cmd.add_argument("--generated-at") + review_waivers_cmd.add_argument("--add-waiver", action="store_true") + review_waivers_cmd.add_argument( + "--gate-key", + choices=[ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "registry-audit", + "release-notes", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "permission-gates", + "permission-runtime", + ], + ) + review_waivers_cmd.add_argument( + "--decision", + choices=["accepted-risk", "false-positive", "temporary-exception"], + default="accepted-risk", + ) + review_waivers_cmd.add_argument("--reviewer") + review_waivers_cmd.add_argument("--reason") + review_waivers_cmd.add_argument("--expires-at") + review_waivers_cmd.add_argument("--created-at") + review_waivers_cmd.add_argument("--evidence") + review_waivers_cmd.add_argument("--scope", default="current-release") + review_waivers_cmd.set_defaults(func=command_review_waivers) + + review_annotations_cmd = subparsers.add_parser( + "review-annotations", + help="Render or update inline reviewer annotations for Review Studio gates and source paths.", + ) + review_annotations_cmd.add_argument("skill_dir", nargs="?", default=".") + review_annotations_cmd.add_argument("--annotations-json") + review_annotations_cmd.add_argument("--output-json") + review_annotations_cmd.add_argument("--output-md") + review_annotations_cmd.add_argument("--write-template", action="store_true") + review_annotations_cmd.add_argument("--add-annotation", action="store_true") + review_annotations_cmd.add_argument("--annotation-id") + review_annotations_cmd.add_argument( + "--gate-key", + choices=[ + "context-budget", + "intent-canvas", + "operations-loop", + "output-lab", + "registry-audit", + "release-notes", + "review-waivers", + "runtime-matrix", + "skill-atlas", + "trigger-lab", + "trust-report", + "permission-gates", + "permission-runtime", + ], + ) + review_annotations_cmd.add_argument("--target-path") + review_annotations_cmd.add_argument("--line", type=int) + review_annotations_cmd.add_argument("--severity", choices=["blocker", "info", "note", "warning"], default="note") + review_annotations_cmd.add_argument("--status", choices=["deferred", "open", "resolved"], default="open") + review_annotations_cmd.add_argument("--reviewer") + review_annotations_cmd.add_argument("--created-at") + review_annotations_cmd.add_argument("--body") + review_annotations_cmd.add_argument("--suggested-action") + review_annotations_cmd.add_argument("--evidence") + review_annotations_cmd.set_defaults(func=command_review_annotations) + baseline_compare_cmd = subparsers.add_parser( "baseline-compare", help="Render a lightweight with-skill vs baseline comparison across tracked targets.", @@ -1337,12 +1726,49 @@ def build_parser() -> argparse.ArgumentParser: skill_ir_cmd.add_argument("--validate-only", action="store_true") skill_ir_cmd.set_defaults(func=command_skill_ir) + compile_skill_cmd = subparsers.add_parser( + "compile-skill", + help="Compile Skill IR into target-specific semantic contracts.", + ) + compile_skill_cmd.add_argument("skill_dir", nargs="?", default=".") + compile_skill_cmd.add_argument("--target", action="append") + compile_skill_cmd.add_argument("--output-json") + compile_skill_cmd.add_argument("--output-md") + compile_skill_cmd.add_argument("--generated-at") + compile_skill_cmd.set_defaults(func=command_compile_skill) + output_eval_cmd = subparsers.add_parser("output-eval", help="Run Output Eval Lab assertion grading.") output_eval_cmd.add_argument("--cases") output_eval_cmd.add_argument("--output-json") output_eval_cmd.add_argument("--output-md") + output_eval_cmd.add_argument("--blind-pack-json") + output_eval_cmd.add_argument("--blind-pack-md") + output_eval_cmd.add_argument("--blind-answer-key-json") output_eval_cmd.set_defaults(func=command_output_eval) + output_execution_cmd = subparsers.add_parser( + "output-exec", + help="Record output-eval execution evidence, timing, and token usage.", + ) + output_execution_cmd.add_argument("--cases") + output_execution_cmd.add_argument("--output-json") + output_execution_cmd.add_argument("--output-md") + output_execution_cmd.add_argument("--runner-command") + output_execution_cmd.add_argument("--timeout-seconds", type=float) + output_execution_cmd.set_defaults(func=command_output_execution) + + output_review_cmd = subparsers.add_parser( + "output-review", + help="Adjudicate blind A/B output review decisions against the answer key.", + ) + output_review_cmd.add_argument("--blind-pack") + output_review_cmd.add_argument("--answer-key") + output_review_cmd.add_argument("--decisions") + output_review_cmd.add_argument("--output-json") + output_review_cmd.add_argument("--output-md") + output_review_cmd.add_argument("--write-template", action="store_true") + output_review_cmd.set_defaults(func=command_output_review) + conformance_cmd = subparsers.add_parser("conformance", help="Run runtime conformance checks for Skill OS targets.") conformance_cmd.add_argument("skill_dir", nargs="?", default=".") conformance_cmd.add_argument("--target", action="append", choices=["openai", "claude", "agent-skills", "vscode", "generic"]) @@ -1350,6 +1776,17 @@ def build_parser() -> argparse.ArgumentParser: conformance_cmd.add_argument("--output-md") conformance_cmd.set_defaults(func=command_conformance) + runtime_permissions_cmd = subparsers.add_parser( + "runtime-permissions", + help="Probe generated target adapters for runtime permission enforcement metadata.", + ) + runtime_permissions_cmd.add_argument("skill_dir", nargs="?", default=".") + runtime_permissions_cmd.add_argument("--package-dir", default="dist") + runtime_permissions_cmd.add_argument("--target", action="append", choices=["openai", "claude", "generic"]) + runtime_permissions_cmd.add_argument("--output-json") + runtime_permissions_cmd.add_argument("--output-md") + runtime_permissions_cmd.set_defaults(func=command_runtime_permissions) + trust_cmd = subparsers.add_parser("trust", help="Run trust and security checks for a skill package.") trust_cmd.add_argument("skill_dir", nargs="?", default=".") trust_cmd.add_argument("--output-json") @@ -1365,6 +1802,14 @@ def build_parser() -> argparse.ArgumentParser: skill_atlas_cmd.add_argument("--today") skill_atlas_cmd.set_defaults(func=command_skill_atlas) + registry_audit_cmd = subparsers.add_parser("registry-audit", help="Build and audit Skill OS registry package metadata.") + registry_audit_cmd.add_argument("skill_dir", nargs="?", default=".") + registry_audit_cmd.add_argument("--registry-dir") + registry_audit_cmd.add_argument("--output-json") + registry_audit_cmd.add_argument("--output-md") + registry_audit_cmd.add_argument("--generated-at") + registry_audit_cmd.set_defaults(func=command_registry_audit) + package_cmd = subparsers.add_parser("package", help="Export compatibility artifacts for selected targets.") package_cmd.add_argument("skill_dir", nargs="?", default=".") package_cmd.add_argument("--platform", action="append") @@ -1373,6 +1818,35 @@ def build_parser() -> argparse.ArgumentParser: package_cmd.add_argument("--zip", action="store_true") package_cmd.set_defaults(func=command_package) + package_verify_cmd = subparsers.add_parser("package-verify", help="Verify generated package artifacts, archive safety, and registry parity.") + package_verify_cmd.add_argument("skill_dir", nargs="?", default=".") + package_verify_cmd.add_argument("--package-dir", default="dist") + package_verify_cmd.add_argument("--expectations", default="evals/packaging_expectations.json") + package_verify_cmd.add_argument("--registry-json", default="reports/registry_audit.json") + package_verify_cmd.add_argument("--output-json") + package_verify_cmd.add_argument("--output-md") + package_verify_cmd.add_argument("--require-zip", action="store_true") + package_verify_cmd.add_argument("--generated-at") + package_verify_cmd.set_defaults(func=command_package_verify) + + install_simulate_cmd = subparsers.add_parser("install-simulate", help="Simulate installing a generated package into a temporary local skill root.") + install_simulate_cmd.add_argument("skill_dir", nargs="?", default=".") + install_simulate_cmd.add_argument("--package-dir", default="dist") + install_simulate_cmd.add_argument("--install-root") + install_simulate_cmd.add_argument("--output-json") + install_simulate_cmd.add_argument("--output-md") + install_simulate_cmd.add_argument("--generated-at") + install_simulate_cmd.set_defaults(func=command_install_simulate) + + upgrade_check_cmd = subparsers.add_parser("upgrade-check", help="Compare current and previous registry package metadata for upgrade readiness.") + upgrade_check_cmd.add_argument("skill_dir", nargs="?", default=".") + upgrade_check_cmd.add_argument("--previous-package-json", required=True) + upgrade_check_cmd.add_argument("--current-package-json", default="reports/registry_audit.json") + upgrade_check_cmd.add_argument("--output-json") + upgrade_check_cmd.add_argument("--output-md") + upgrade_check_cmd.add_argument("--generated-at") + upgrade_check_cmd.set_defaults(func=command_upgrade_check) + test_cmd = subparsers.add_parser("test", help="Run a Makefile test target.") test_cmd.add_argument("--target", default="test") test_cmd.set_defaults(func=command_test) diff --git a/security/network_policy.json b/security/network_policy.json new file mode 100644 index 0000000..b2eb87d --- /dev/null +++ b/security/network_policy.json @@ -0,0 +1,37 @@ +{ + "schema_version": "1.0", + "default_policy": { + "require_https": true, + "custom_hosts": "deny-by-default", + "review_rule": "Network-capable scripts must declare allowed_hosts and justify any explicit custom-host override." + }, + "scripts": { + "scripts/check_update.py": { + "purpose": "Check the public yao-meta-skill GitHub source for a newer VERSION or manifest version.", + "allowed_hosts": [ + "raw.githubusercontent.com" + ], + "allowed_path_prefixes": [ + "/yaojingang/yao-meta-skill/" + ], + "requires_https": true, + "custom_url_policy": "explicit-opt-in-only via --allow-custom-update-url or YAO_ALLOW_CUSTOM_UPDATE_URL=1", + "timeout_seconds": 3, + "auth": "none" + }, + "scripts/github_benchmark_scan.py": { + "purpose": "Search public GitHub repositories and fetch README excerpts as benchmark references for skill creation.", + "allowed_hosts": [ + "api.github.com" + ], + "allowed_path_prefixes": [ + "/search/repositories", + "/repos/" + ], + "requires_https": true, + "custom_url_policy": "not supported", + "timeout_seconds": 15, + "auth": "optional GITHUB_TOKEN or GH_TOKEN for GitHub API rate limits" + } + } +} diff --git a/security/network_policy.md b/security/network_policy.md new file mode 100644 index 0000000..db29578 --- /dev/null +++ b/security/network_policy.md @@ -0,0 +1,12 @@ +# Network Policy + +Network-capable scripts are allowed only when their outbound boundary is explicit and reviewable. + +Current policy: + +- `scripts/check_update.py` may read version metadata from `raw.githubusercontent.com/yaojingang/yao-meta-skill/` over HTTPS. +- `scripts/github_benchmark_scan.py` may call the GitHub REST API at `api.github.com` over HTTPS. +- Custom update URLs are denied by default and require explicit operator opt-in plus reviewer awareness. +- New network-capable scripts must be added to `security/network_policy.json` before a governed or team-distributed release. + +The trust checker compares HTTPS URL literals in scripts with `security/network_policy.json`. Missing or mismatched host declarations are reported as warnings. diff --git a/security/permission_policy.json b/security/permission_policy.json new file mode 100644 index 0000000..ec54311 --- /dev/null +++ b/security/permission_policy.json @@ -0,0 +1,55 @@ +{ + "schema_version": "1.0", + "reviewed_at": "2026-06-13", + "capabilities": { + "network": { + "decision": "approved", + "reviewer": "Yao Team", + "scope": "Only scripts declared in security/network_policy.json may perform outbound requests.", + "reason": "Network use is bounded to public GitHub metadata and benchmark discovery, with allowed hosts reviewed in security/network_policy.json.", + "expires_at": "2026-09-30", + "evidence": [ + "security/network_policy.json", + "reports/security_trust_report.md" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + } + }, + "file_write": { + "decision": "approved", + "reviewer": "Yao Team", + "scope": "Local generated reports, registry metadata, temporary test outputs, and package artifacts under the skill workspace.", + "reason": "File writes are required to create auditable reports, compiled target contracts, package artifacts, and local test fixtures; package safety checks exclude generated temp outputs from archives.", + "expires_at": "2026-09-30", + "evidence": [ + "reports/package_verification.md", + "reports/install_simulation.md", + "security/permission_policy.md" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + } + }, + "subprocess": { + "decision": "approved", + "reviewer": "Yao Team", + "scope": "Local deterministic validation, packaging, install simulation, and CI orchestration commands.", + "reason": "Subprocess use is limited to local verification and packaging flows that are surfaced through CLI help smoke tests and CI targets.", + "expires_at": "2026-09-30", + "evidence": [ + "reports/security_trust_report.md", + "reports/review-studio.html" + ], + "target_enforcement": { + "openai": "metadata-only permission_contract in openai.yaml plus adapter.json", + "claude": "adapter.json target_permission_contract and README note", + "generic": "adapter.json target_permission_contract" + } + } + } +} diff --git a/security/permission_policy.md b/security/permission_policy.md new file mode 100644 index 0000000..a0d4c2d --- /dev/null +++ b/security/permission_policy.md @@ -0,0 +1,31 @@ +# Permission Policy + +Permission metadata is generated from `reports/security_trust_report.json` and embedded into target adapters at package time. Governed releases also require explicit reviewer approvals in `security/permission_policy.json`. + +Current v0 capabilities: + +- `network`: scripts that perform outbound requests, bounded by `security/network_policy.json`. +- `file_write`: scripts that create, modify, delete, copy, move, or archive local files. +- `subprocess`: scripts that spawn local commands. +- `interactive`: scripts that prompt for user input. + +Each packaged target must carry: + +- `permission_contract`: platform-neutral capability summary and evidence path. +- `target_permission_contract`: target-specific representation, reviewer note, and native-enforcement status. + +Each high-permission approval must include: + +- `decision: approved` +- `reviewer` +- `scope` +- `reason` +- `expires_at` +- `evidence` +- `target_enforcement` for OpenAI, Claude, and generic packages + +Review Studio exposes this as the `permission-gates` gate. Missing, invalid, or expired approvals block governed mode and stay reviewer-visible in lighter modes. + +After packaging, run `python3 scripts/probe_runtime_permissions.py . --package-dir dist` to generate `reports/runtime_permission_probes.md` and `reports/runtime_permission_probes.json`. Review Studio exposes that evidence as the `permission-runtime` gate, separate from approval governance. A passing probe means target adapters make permission handling explicit and auditable; it does not claim client-native enforcement when the target only supports metadata fallback. + +Current targets preserve permission semantics as metadata. They do not enforce permissions at runtime. Any future native installer or client integration must map these capabilities into that client's permission model rather than dropping them. diff --git a/security/script_policy.md b/security/script_policy.md index f1fd4d4..b687f9c 100644 --- a/security/script_policy.md +++ b/security/script_policy.md @@ -6,6 +6,8 @@ Minimum expectations: - expose a command-line entrypoint or clear function boundary - prefer `argparse` and `--help` for operator-facing scripts +- keep `python3 scripts/name.py --help` safe, non-mutating, and fast enough for trust smoke checks - avoid hidden interactive prompts unless the command is explicitly interactive - state network or filesystem side effects in nearby docs or trust metadata +- declare allowed network hosts in `security/network_policy.json` before team distribution - emit structured JSON when used by `scripts/yao.py` diff --git a/security/trust_policy.md b/security/trust_policy.md index c1e0205..4b2ed60 100644 --- a/security/trust_policy.md +++ b/security/trust_policy.md @@ -6,9 +6,16 @@ Governed or team-distributed skills should produce a trust report before release - secret exposure risk - scripts that lack a visible CLI/help surface +- execution-level CLI `--help` smoke failures - interactive or network-capable scripts +- bounded host policy for each network-capable script +- reviewer approval for each high-permission capability - dependency pinning status - runtime trust metadata - package integrity hash High-risk secrets or unrestricted remote inline execution block governed release. + +Network-capable scripts without a matching `security/network_policy.json` entry remain reviewer-visible warnings until the host boundary is declared. + +High-permission capabilities without valid `security/permission_policy.json` approvals block governed mode. Required approvals must name the reviewer, scope, reason, expiry, evidence, and target-enforcement mapping. diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index 6ee8555..bc03202 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -2,7 +2,7 @@ "schema_version": "2.0.0", "name": "yao-meta-skill", "title": "Yao Meta Skill", - "job_to_be_done": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "job_to_be_done": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.", "trigger_surface": { "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "should_trigger": [ @@ -24,8 +24,8 @@ "Draft a blog title for this article.", "Rewrite this paragraph more clearly, but do not package anything.", "Explain the difference between a runbook and a workflow.", - "one-off adjacent requests that do not match the recurring job", - "private local material that was not intentionally included" + "one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill", + "general code review or debugging unless the user is packaging that workflow as a skill" ], "edge_cases": [ "Create a one-off prompt for this task.", @@ -76,6 +76,7 @@ "references": [ "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", "references/governance.md", @@ -96,20 +97,24 @@ "references/regression-cause-taxonomy.md", "references/resource-boundaries.md", "references/review-studio-method.md", + "references/review-waiver-method.md", "references/runtime-conformance-method.md", "references/skill-archetypes.md", "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", "references/trust-security-method.md" ], "scripts": [ + "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", + "scripts/compile_skill.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", @@ -120,8 +125,12 @@ "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", "scripts/render_context_reports.py", @@ -137,8 +146,10 @@ "scripts/render_reference_scan.py", "scripts/render_reference_synthesis.py", "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", "scripts/render_review_studio.py", "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", @@ -147,13 +158,17 @@ "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", "scripts/skill_report_charts.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/upgrade_check.py", "scripts/validate_skill.py", + "scripts/verify_package.py", "scripts/yao.py" ], "assets": [ @@ -181,10 +196,21 @@ "reports/skill-overview.html", "reports/output_quality_scorecard.json", "reports/output_quality_scorecard.md", + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.json", + "reports/output_blind_review_pack.md", + "reports/output_blind_answer_key.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "reports/review_annotations.json", + "reports/review_annotations.md", "reports/conformance_matrix.json", "reports/conformance_matrix.md", "reports/security_trust_report.json", "reports/security_trust_report.md", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", "reports/skill_atlas.json", "reports/skill_atlas.html", "reports/skill-os-2-review.md", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 1533c9f..e1a7105 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -19,11 +19,13 @@ "agent-skills-compatible" ], "resources": [ + "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", "scripts/build_skill_atlas.py", "scripts/check_update.py", "scripts/ci_test.py", "scripts/collect_feedback.py", + "scripts/compile_skill.py", "scripts/context_sizer.py", "scripts/create_iteration_snapshot.py", "scripts/cross_packager.py", @@ -34,8 +36,12 @@ "scripts/init_skill.py", "scripts/judge_blind_eval.py", "scripts/lint_skill.py", + "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/probe_runtime_permissions.py", "scripts/promotion_checker.py", + "scripts/registry_audit.py", + "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", "scripts/render_context_reports.py", @@ -51,8 +57,10 @@ "scripts/render_reference_scan.py", "scripts/render_reference_synthesis.py", "scripts/render_regression_history.py", + "scripts/render_review_annotations.py", "scripts/render_review_studio.py", "scripts/render_review_viewer.py", + "scripts/render_review_waivers.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", "scripts/render_system_model.py", @@ -61,16 +69,21 @@ "scripts/run_description_optimization_suite.py", "scripts/run_eval_suite.py", "scripts/run_output_eval.py", + "scripts/run_output_execution.py", + "scripts/simulate_install.py", "scripts/skill_report_charts.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", "scripts/sync_local_install.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/upgrade_check.py", "scripts/validate_skill.py", + "scripts/verify_package.py", "scripts/yao.py", "references/artifact-design-doctrine.md", "references/authoring-discipline.md", + "references/distribution-registry-method.md", "references/eval-playbook.md", "references/gate-selection.md", "references/governance.md", @@ -91,12 +104,14 @@ "references/regression-cause-taxonomy.md", "references/resource-boundaries.md", "references/review-studio-method.md", + "references/review-waiver-method.md", "references/runtime-conformance-method.md", "references/skill-archetypes.md", "references/skill-atlas-method.md", "references/skill-engineering-method.md", "references/skill-ir-method.md", "references/systems-thinking-doctrine.md", + "references/telemetry-drift-method.md", "references/trust-security-method.md", "templates/basic_skill.md.j2", "templates/complex_skill.md.j2" @@ -120,7 +135,10 @@ "team", "transcripts", "workflows" - ] + ], + "atlas_scope": "release", + "actionable": true, + "scope_reason": "default release-actionable skill" }, { "name": "release-orchestrator", @@ -175,7 +193,10 @@ "stakeholder", "theory", "workflow" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -211,7 +232,10 @@ "states", "ui", "ux" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -231,7 +255,10 @@ "frontend", "merge", "review" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -266,7 +293,10 @@ "security", "states", "ui" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "incident-command-governor", @@ -304,7 +334,10 @@ "run", "severity", "standardize" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "note-cleanup", @@ -338,7 +371,10 @@ "separate", "structured", "summaries" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "frontend-review", @@ -375,7 +411,10 @@ "ui", "unsafe", "ux" - ] + ], + "atlas_scope": "example", + "actionable": false, + "scope_reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." }, { "name": "geo-ranking-article-generator", @@ -432,7 +471,10 @@ "write", "writing", "生成式引擎优化" - ] + ], + "atlas_scope": "embedded-generated-skill", + "actionable": false, + "scope_reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." }, { "name": "invalid-governance-skill", @@ -451,7 +493,10 @@ "governance", "test", "validation" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." }, { "name": "broken-yaml-skill", @@ -472,7 +517,10 @@ "packager", "testing", "yaml" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." }, { "name": "broken-skill", @@ -492,15 +540,23 @@ "fixture", "packager", "testing" - ] + ], + "atlas_scope": "test-fixture", + "actionable": false, + "scope_reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." } ], "summary": { "skill_count": 12, + "actionable_skill_count": 1, "route_collision_count": 5, + "actionable_route_collision_count": 0, "owner_gap_count": 9, + "actionable_owner_gap_count": 0, "stale_count": 10, + "actionable_stale_count": 0, "shared_resource_count": 0, - "no_route_opportunity_count": 0 + "no_route_opportunity_count": 0, + "non_actionable_issue_count": 24 } } diff --git a/skill_atlas/owner_review_gaps.json b/skill_atlas/owner_review_gaps.json index ec951f4..e69e911 100644 --- a/skill_atlas/owner_review_gaps.json +++ b/skill_atlas/owner_review_gaps.json @@ -6,7 +6,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -15,7 +17,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -24,7 +28,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "note-cleanup", @@ -33,7 +39,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "frontend-review", @@ -42,7 +50,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "example" }, { "name": "geo-ranking-article-generator", @@ -51,7 +61,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "embedded-generated-skill" }, { "name": "invalid-governance-skill", @@ -59,7 +71,9 @@ "missing": [ "owner", "review_cadence" - ] + ], + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-yaml-skill", @@ -68,7 +82,9 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-skill", @@ -77,6 +93,8 @@ "owner", "review_cadence", "maturity" - ] + ], + "actionable": false, + "scope": "test-fixture" } ] diff --git a/skill_atlas/policy.json b/skill_atlas/policy.json new file mode 100644 index 0000000..f903e58 --- /dev/null +++ b/skill_atlas/policy.json @@ -0,0 +1,23 @@ +{ + "schema_version": "1.0", + "scope_rules": [ + { + "path_prefix": "examples/", + "scope": "example", + "actionable": false, + "reason": "Generated examples and evolution snapshots are portfolio evidence, not release-actionable library skills for the root package." + }, + { + "path_prefix": "tests/fixtures/", + "scope": "test-fixture", + "actionable": false, + "reason": "Fixtures intentionally include invalid or broken skills to verify validators and packagers." + }, + { + "path_prefix": "geo-ranking-article-generator", + "scope": "embedded-generated-skill", + "actionable": false, + "reason": "This nested generated skill is retained as local evidence and should not gate the root Meta Skill release." + } + ] +} diff --git a/skill_atlas/route_overlap_matrix.csv b/skill_atlas/route_overlap_matrix.csv index 33d5a89..6f598a2 100644 --- a/skill_atlas/route_overlap_matrix.csv +++ b/skill_atlas/route_overlap_matrix.csv @@ -1,67 +1,67 @@ -skill_a,skill_b,path_a,path_b,score,status -yao-meta-skill,release-orchestrator,.,examples/complex-release-orchestrator/generated-skill,0.061,clear -yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/final,0.054,clear -yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v0,0.0,clear -yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v1,0.056,clear -yao-meta-skill,incident-command-governor,.,examples/governed-incident-command/generated-skill,0.067,clear -yao-meta-skill,note-cleanup,.,examples/simple-note-cleanup/generated-skill,0.088,clear -yao-meta-skill,frontend-review,.,examples/team-frontend-review/generated-skill,0.056,clear -yao-meta-skill,geo-ranking-article-generator,.,geo-ranking-article-generator,0.038,clear -yao-meta-skill,invalid-governance-skill,.,tests/fixtures/governance_invalid_manifest,0.0,clear -yao-meta-skill,broken-yaml-skill,.,tests/fixtures/package_invalid_yaml,0.0,clear -yao-meta-skill,broken-skill,.,tests/fixtures/package_missing_interface_field,0.0,clear -release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/final,0.058,clear -release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/v0,0.026,clear -release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/v1,0.059,clear -release-orchestrator,incident-command-governor,examples/complex-release-orchestrator/generated-skill,examples/governed-incident-command/generated-skill,0.116,clear -release-orchestrator,note-cleanup,examples/complex-release-orchestrator/generated-skill,examples/simple-note-cleanup/generated-skill,0.06,clear -release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/team-frontend-review/generated-skill,0.059,clear -release-orchestrator,geo-ranking-article-generator,examples/complex-release-orchestrator/generated-skill,geo-ranking-article-generator,0.094,clear -release-orchestrator,invalid-governance-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear -release-orchestrator,broken-yaml-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear -release-orchestrator,broken-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear -frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/evolution-frontend-review/v0,0.13,clear -frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/evolution-frontend-review/v1,0.708,collision -frontend-review,incident-command-governor,examples/evolution-frontend-review/final,examples/governed-incident-command/generated-skill,0.094,clear -frontend-review,note-cleanup,examples/evolution-frontend-review/final,examples/simple-note-cleanup/generated-skill,0.053,clear -frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/team-frontend-review/generated-skill,0.64,collision -frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/final,geo-ranking-article-generator,0.036,clear -frontend-review,invalid-governance-skill,examples/evolution-frontend-review/final,tests/fixtures/governance_invalid_manifest,0.0,clear -frontend-review,broken-yaml-skill,examples/evolution-frontend-review/final,tests/fixtures/package_invalid_yaml,0.0,clear -frontend-review,broken-skill,examples/evolution-frontend-review/final,tests/fixtures/package_missing_interface_field,0.0,clear -frontend-review,frontend-review,examples/evolution-frontend-review/v0,examples/evolution-frontend-review/v1,0.136,clear -frontend-review,incident-command-governor,examples/evolution-frontend-review/v0,examples/governed-incident-command/generated-skill,0.056,clear -frontend-review,note-cleanup,examples/evolution-frontend-review/v0,examples/simple-note-cleanup/generated-skill,0.0,clear -frontend-review,frontend-review,examples/evolution-frontend-review/v0,examples/team-frontend-review/generated-skill,0.136,clear -frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/v0,geo-ranking-article-generator,0.0,clear -frontend-review,invalid-governance-skill,examples/evolution-frontend-review/v0,tests/fixtures/governance_invalid_manifest,0.0,clear -frontend-review,broken-yaml-skill,examples/evolution-frontend-review/v0,tests/fixtures/package_invalid_yaml,0.0,clear -frontend-review,broken-skill,examples/evolution-frontend-review/v0,tests/fixtures/package_missing_interface_field,0.0,clear -frontend-review,incident-command-governor,examples/evolution-frontend-review/v1,examples/governed-incident-command/generated-skill,0.097,clear -frontend-review,note-cleanup,examples/evolution-frontend-review/v1,examples/simple-note-cleanup/generated-skill,0.054,clear -frontend-review,frontend-review,examples/evolution-frontend-review/v1,examples/team-frontend-review/generated-skill,0.538,collision -frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/v1,geo-ranking-article-generator,0.037,clear -frontend-review,invalid-governance-skill,examples/evolution-frontend-review/v1,tests/fixtures/governance_invalid_manifest,0.0,clear -frontend-review,broken-yaml-skill,examples/evolution-frontend-review/v1,tests/fixtures/package_invalid_yaml,0.0,clear -frontend-review,broken-skill,examples/evolution-frontend-review/v1,tests/fixtures/package_missing_interface_field,0.0,clear -incident-command-governor,note-cleanup,examples/governed-incident-command/generated-skill,examples/simple-note-cleanup/generated-skill,0.065,clear -incident-command-governor,frontend-review,examples/governed-incident-command/generated-skill,examples/team-frontend-review/generated-skill,0.133,clear -incident-command-governor,geo-ranking-article-generator,examples/governed-incident-command/generated-skill,geo-ranking-article-generator,0.042,clear -incident-command-governor,invalid-governance-skill,examples/governed-incident-command/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear -incident-command-governor,broken-yaml-skill,examples/governed-incident-command/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear -incident-command-governor,broken-skill,examples/governed-incident-command/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear -note-cleanup,frontend-review,examples/simple-note-cleanup/generated-skill,examples/team-frontend-review/generated-skill,0.054,clear -note-cleanup,geo-ranking-article-generator,examples/simple-note-cleanup/generated-skill,geo-ranking-article-generator,0.038,clear -note-cleanup,invalid-governance-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear -note-cleanup,broken-yaml-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear -note-cleanup,broken-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear -frontend-review,geo-ranking-article-generator,examples/team-frontend-review/generated-skill,geo-ranking-article-generator,0.037,clear -frontend-review,invalid-governance-skill,examples/team-frontend-review/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear -frontend-review,broken-yaml-skill,examples/team-frontend-review/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear -frontend-review,broken-skill,examples/team-frontend-review/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear -geo-ranking-article-generator,invalid-governance-skill,geo-ranking-article-generator,tests/fixtures/governance_invalid_manifest,0.0,clear -geo-ranking-article-generator,broken-yaml-skill,geo-ranking-article-generator,tests/fixtures/package_invalid_yaml,0.0,clear -geo-ranking-article-generator,broken-skill,geo-ranking-article-generator,tests/fixtures/package_missing_interface_field,0.0,clear -invalid-governance-skill,broken-yaml-skill,tests/fixtures/governance_invalid_manifest,tests/fixtures/package_invalid_yaml,0.111,clear -invalid-governance-skill,broken-skill,tests/fixtures/governance_invalid_manifest,tests/fixtures/package_missing_interface_field,0.125,clear -broken-yaml-skill,broken-skill,tests/fixtures/package_invalid_yaml,tests/fixtures/package_missing_interface_field,0.833,collision +skill_a,skill_b,path_a,path_b,score,status,actionable,scope_a,scope_b +yao-meta-skill,release-orchestrator,.,examples/complex-release-orchestrator/generated-skill,0.061,clear,False,release,example +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/final,0.054,clear,False,release,example +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v0,0.0,clear,False,release,example +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v1,0.056,clear,False,release,example +yao-meta-skill,incident-command-governor,.,examples/governed-incident-command/generated-skill,0.067,clear,False,release,example +yao-meta-skill,note-cleanup,.,examples/simple-note-cleanup/generated-skill,0.088,clear,False,release,example +yao-meta-skill,frontend-review,.,examples/team-frontend-review/generated-skill,0.056,clear,False,release,example +yao-meta-skill,geo-ranking-article-generator,.,geo-ranking-article-generator,0.038,clear,False,release,embedded-generated-skill +yao-meta-skill,invalid-governance-skill,.,tests/fixtures/governance_invalid_manifest,0.0,clear,False,release,test-fixture +yao-meta-skill,broken-yaml-skill,.,tests/fixtures/package_invalid_yaml,0.0,clear,False,release,test-fixture +yao-meta-skill,broken-skill,.,tests/fixtures/package_missing_interface_field,0.0,clear,False,release,test-fixture +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/final,0.058,clear,False,example,example +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/v0,0.026,clear,False,example,example +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/v1,0.059,clear,False,example,example +release-orchestrator,incident-command-governor,examples/complex-release-orchestrator/generated-skill,examples/governed-incident-command/generated-skill,0.116,clear,False,example,example +release-orchestrator,note-cleanup,examples/complex-release-orchestrator/generated-skill,examples/simple-note-cleanup/generated-skill,0.06,clear,False,example,example +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/team-frontend-review/generated-skill,0.059,clear,False,example,example +release-orchestrator,geo-ranking-article-generator,examples/complex-release-orchestrator/generated-skill,geo-ranking-article-generator,0.094,clear,False,example,embedded-generated-skill +release-orchestrator,invalid-governance-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear,False,example,test-fixture +release-orchestrator,broken-yaml-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear,False,example,test-fixture +release-orchestrator,broken-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear,False,example,test-fixture +frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/evolution-frontend-review/v0,0.13,clear,False,example,example +frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/evolution-frontend-review/v1,0.708,collision,False,example,example +frontend-review,incident-command-governor,examples/evolution-frontend-review/final,examples/governed-incident-command/generated-skill,0.094,clear,False,example,example +frontend-review,note-cleanup,examples/evolution-frontend-review/final,examples/simple-note-cleanup/generated-skill,0.053,clear,False,example,example +frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/team-frontend-review/generated-skill,0.64,collision,False,example,example +frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/final,geo-ranking-article-generator,0.036,clear,False,example,embedded-generated-skill +frontend-review,invalid-governance-skill,examples/evolution-frontend-review/final,tests/fixtures/governance_invalid_manifest,0.0,clear,False,example,test-fixture +frontend-review,broken-yaml-skill,examples/evolution-frontend-review/final,tests/fixtures/package_invalid_yaml,0.0,clear,False,example,test-fixture +frontend-review,broken-skill,examples/evolution-frontend-review/final,tests/fixtures/package_missing_interface_field,0.0,clear,False,example,test-fixture +frontend-review,frontend-review,examples/evolution-frontend-review/v0,examples/evolution-frontend-review/v1,0.136,clear,False,example,example +frontend-review,incident-command-governor,examples/evolution-frontend-review/v0,examples/governed-incident-command/generated-skill,0.056,clear,False,example,example +frontend-review,note-cleanup,examples/evolution-frontend-review/v0,examples/simple-note-cleanup/generated-skill,0.0,clear,False,example,example +frontend-review,frontend-review,examples/evolution-frontend-review/v0,examples/team-frontend-review/generated-skill,0.136,clear,False,example,example +frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/v0,geo-ranking-article-generator,0.0,clear,False,example,embedded-generated-skill +frontend-review,invalid-governance-skill,examples/evolution-frontend-review/v0,tests/fixtures/governance_invalid_manifest,0.0,clear,False,example,test-fixture +frontend-review,broken-yaml-skill,examples/evolution-frontend-review/v0,tests/fixtures/package_invalid_yaml,0.0,clear,False,example,test-fixture +frontend-review,broken-skill,examples/evolution-frontend-review/v0,tests/fixtures/package_missing_interface_field,0.0,clear,False,example,test-fixture +frontend-review,incident-command-governor,examples/evolution-frontend-review/v1,examples/governed-incident-command/generated-skill,0.097,clear,False,example,example +frontend-review,note-cleanup,examples/evolution-frontend-review/v1,examples/simple-note-cleanup/generated-skill,0.054,clear,False,example,example +frontend-review,frontend-review,examples/evolution-frontend-review/v1,examples/team-frontend-review/generated-skill,0.538,collision,False,example,example +frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/v1,geo-ranking-article-generator,0.037,clear,False,example,embedded-generated-skill +frontend-review,invalid-governance-skill,examples/evolution-frontend-review/v1,tests/fixtures/governance_invalid_manifest,0.0,clear,False,example,test-fixture +frontend-review,broken-yaml-skill,examples/evolution-frontend-review/v1,tests/fixtures/package_invalid_yaml,0.0,clear,False,example,test-fixture +frontend-review,broken-skill,examples/evolution-frontend-review/v1,tests/fixtures/package_missing_interface_field,0.0,clear,False,example,test-fixture +incident-command-governor,note-cleanup,examples/governed-incident-command/generated-skill,examples/simple-note-cleanup/generated-skill,0.065,clear,False,example,example +incident-command-governor,frontend-review,examples/governed-incident-command/generated-skill,examples/team-frontend-review/generated-skill,0.133,clear,False,example,example +incident-command-governor,geo-ranking-article-generator,examples/governed-incident-command/generated-skill,geo-ranking-article-generator,0.042,clear,False,example,embedded-generated-skill +incident-command-governor,invalid-governance-skill,examples/governed-incident-command/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear,False,example,test-fixture +incident-command-governor,broken-yaml-skill,examples/governed-incident-command/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear,False,example,test-fixture +incident-command-governor,broken-skill,examples/governed-incident-command/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear,False,example,test-fixture +note-cleanup,frontend-review,examples/simple-note-cleanup/generated-skill,examples/team-frontend-review/generated-skill,0.054,clear,False,example,example +note-cleanup,geo-ranking-article-generator,examples/simple-note-cleanup/generated-skill,geo-ranking-article-generator,0.038,clear,False,example,embedded-generated-skill +note-cleanup,invalid-governance-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear,False,example,test-fixture +note-cleanup,broken-yaml-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear,False,example,test-fixture +note-cleanup,broken-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear,False,example,test-fixture +frontend-review,geo-ranking-article-generator,examples/team-frontend-review/generated-skill,geo-ranking-article-generator,0.037,clear,False,example,embedded-generated-skill +frontend-review,invalid-governance-skill,examples/team-frontend-review/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear,False,example,test-fixture +frontend-review,broken-yaml-skill,examples/team-frontend-review/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear,False,example,test-fixture +frontend-review,broken-skill,examples/team-frontend-review/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear,False,example,test-fixture +geo-ranking-article-generator,invalid-governance-skill,geo-ranking-article-generator,tests/fixtures/governance_invalid_manifest,0.0,clear,False,embedded-generated-skill,test-fixture +geo-ranking-article-generator,broken-yaml-skill,geo-ranking-article-generator,tests/fixtures/package_invalid_yaml,0.0,clear,False,embedded-generated-skill,test-fixture +geo-ranking-article-generator,broken-skill,geo-ranking-article-generator,tests/fixtures/package_missing_interface_field,0.0,clear,False,embedded-generated-skill,test-fixture +invalid-governance-skill,broken-yaml-skill,tests/fixtures/governance_invalid_manifest,tests/fixtures/package_invalid_yaml,0.111,clear,False,test-fixture,test-fixture +invalid-governance-skill,broken-skill,tests/fixtures/governance_invalid_manifest,tests/fixtures/package_missing_interface_field,0.125,clear,False,test-fixture,test-fixture +broken-yaml-skill,broken-skill,tests/fixtures/package_invalid_yaml,tests/fixtures/package_missing_interface_field,0.833,collision,False,test-fixture,test-fixture diff --git a/skill_atlas/stale_skills.json b/skill_atlas/stale_skills.json index 14361a6..6602e82 100644 --- a/skill_atlas/stale_skills.json +++ b/skill_atlas/stale_skills.json @@ -2,53 +2,73 @@ { "name": "frontend-review", "path": "examples/evolution-frontend-review/final", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/evolution-frontend-review/v0", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/evolution-frontend-review/v1", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", "age_days": 74, - "allowed_days": 31 + "allowed_days": 31, + "actionable": false, + "scope": "example" }, { "name": "note-cleanup", "path": "examples/simple-note-cleanup/generated-skill", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "frontend-review", "path": "examples/team-frontend-review/generated-skill", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "example" }, { "name": "geo-ranking-article-generator", "path": "geo-ranking-article-generator", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "embedded-generated-skill" }, { "name": "invalid-governance-skill", "path": "tests/fixtures/governance_invalid_manifest", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-yaml-skill", "path": "tests/fixtures/package_invalid_yaml", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" }, { "name": "broken-skill", "path": "tests/fixtures/package_missing_interface_field", - "reason": "missing updated_at" + "reason": "missing updated_at", + "actionable": false, + "scope": "test-fixture" } ] diff --git a/tests/snapshots/claude_adapter.json b/tests/snapshots/claude_adapter.json index 35e4975..5bf0ec1 100644 --- a/tests/snapshots/claude_adapter.json +++ b/tests/snapshots/claude_adapter.json @@ -15,6 +15,12 @@ "ir_source", "ir_schema_version", "semantic_contract", - "semantic_parity" + "semantic_parity", + "compiler", + "compiled_contract", + "permission_contract", + "target_permission_contract", + "target_native_contract", + "target_transform" ] } diff --git a/tests/snapshots/generic_adapter.json b/tests/snapshots/generic_adapter.json index cb976c5..542d8b6 100644 --- a/tests/snapshots/generic_adapter.json +++ b/tests/snapshots/generic_adapter.json @@ -15,6 +15,12 @@ "ir_source", "ir_schema_version", "semantic_contract", - "semantic_parity" + "semantic_parity", + "compiler", + "compiled_contract", + "permission_contract", + "target_permission_contract", + "target_native_contract", + "target_transform" ] } diff --git a/tests/snapshots/openai_adapter.json b/tests/snapshots/openai_adapter.json index 3a3180e..8d63916 100644 --- a/tests/snapshots/openai_adapter.json +++ b/tests/snapshots/openai_adapter.json @@ -15,6 +15,12 @@ "ir_source", "ir_schema_version", "semantic_contract", - "semantic_parity" + "semantic_parity", + "compiler", + "compiled_contract", + "permission_contract", + "target_permission_contract", + "target_native_contract", + "target_transform" ] } diff --git a/tests/verify_adapter_snapshots.py b/tests/verify_adapter_snapshots.py index a8d3ea7..b390ba7 100644 --- a/tests/verify_adapter_snapshots.py +++ b/tests/verify_adapter_snapshots.py @@ -9,6 +9,7 @@ import yaml ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "cross_packager.py" +TRUST_SCRIPT = ROOT / "scripts" / "trust_check.py" EXPECTATIONS = ROOT / "evals" / "packaging_expectations.json" SNAPSHOTS = ROOT / "tests" / "snapshots" TMP = ROOT / "tests" / "tmp_snapshot" @@ -17,6 +18,16 @@ TMP = ROOT / "tests" / "tmp_snapshot" def main() -> None: if TMP.exists(): shutil.rmtree(TMP) + trust_proc = subprocess.run( + [sys.executable, str(TRUST_SCRIPT), str(ROOT)], + cwd=ROOT, + capture_output=True, + text=True, + ) + if trust_proc.returncode != 0: + print(trust_proc.stdout) + print(trust_proc.stderr) + raise SystemExit(trust_proc.returncode) proc = subprocess.run( [ sys.executable, @@ -75,6 +86,54 @@ def main() -> None: failures.append(f"{name}: frontmatter name does not match Skill IR") if parity.get("description_matches_ir") is not True: failures.append(f"{name}: frontmatter description does not match Skill IR") + compiler = adapter.get("compiler", {}) + if compiler.get("name") != "yao-skill-ir-compiler": + failures.append(f"{name}: missing compiler provenance") + compiled = adapter.get("compiled_contract", {}) + if compiled.get("target") != name: + failures.append(f"{name}: compiled contract target mismatch") + if compiled.get("trigger", {}).get("description") != adapter.get("description"): + failures.append(f"{name}: compiled trigger description is not adapter description") + permission = adapter.get("permission_contract", {}) + if permission.get("source") != "reports/security_trust_report.json": + failures.append(f"{name}: permission contract is not sourced from trust report") + if "file_write" not in permission.get("declared_capabilities", []): + failures.append(f"{name}: permission contract does not carry file_write capability") + if permission.get("help_smoke", {}).get("failed_count") != 0: + failures.append(f"{name}: permission contract reports help smoke failures") + target_permission = adapter.get("target_permission_contract", {}) + if target_permission.get("target") != name: + failures.append(f"{name}: target permission contract target mismatch") + if not target_permission.get("representation"): + failures.append(f"{name}: target permission contract lacks representation") + if target_permission.get("capability_counts", {}).get("file_write", 0) <= 0: + failures.append(f"{name}: target permission contract lacks file_write count") + native = adapter.get("target_native_contract", {}) + if native.get("target") != name: + failures.append(f"{name}: target native contract target mismatch") + if not native.get("native_surface"): + failures.append(f"{name}: target native contract lacks native surface") + if not native.get("activation", {}).get("policy"): + failures.append(f"{name}: target native contract lacks activation policy") + if not native.get("resources", {}).get("strategy"): + failures.append(f"{name}: target native contract lacks resource strategy") + if not native.get("scripts", {}).get("strategy"): + failures.append(f"{name}: target native contract lacks script strategy") + if not native.get("permissions", {}).get("enforcement"): + failures.append(f"{name}: target native contract lacks permission enforcement") + if not native.get("review", {}).get("artifacts"): + failures.append(f"{name}: target native contract lacks review artifacts") + transform = adapter.get("target_transform", {}) + if transform.get("target") != name: + failures.append(f"{name}: target transform target mismatch") + if not transform.get("generated_files"): + failures.append(f"{name}: target transform does not declare generated files") + if not transform.get("permission_representation"): + failures.append(f"{name}: target transform does not declare permission representation") + if transform.get("native_surface") != native.get("native_surface"): + failures.append(f"{name}: target transform native surface does not match native contract") + if transform.get("activation_policy") != native.get("activation", {}).get("policy"): + failures.append(f"{name}: target transform activation policy does not match native contract") if not (TMP / snapshot["required_generated_file"]).exists(): failures.append(f"{name}: missing generated file {snapshot['required_generated_file']}") if name == "openai": @@ -83,6 +142,16 @@ def main() -> None: for field in ("canonical_format", "activation_mode", "execution_context", "shell", "trust_level", "remote_inline_execution", "degradation_strategy"): if not compatibility.get(field): failures.append(f"{name}: missing portability metadata in generated openai.yaml: {field}") + yaml_permission = compatibility.get("permission_contract", {}) + if "file_write" not in yaml_permission.get("declared_capabilities", []): + failures.append(f"{name}: generated openai.yaml does not carry permission capabilities") + native_yaml = compatibility.get("native_contract", {}) + if native_yaml.get("native_surface") != native.get("native_surface"): + failures.append(f"{name}: generated openai.yaml does not carry native surface") + if name == "claude": + readme = (TMP / "targets" / "claude" / "README.md").read_text(encoding="utf-8") + if "Native surface:" not in readme or "Activation:" not in readme: + failures.append(f"{name}: generated README does not describe native behavior") report = {"ok": not failures, "failures": failures} print(json.dumps(report, ensure_ascii=False, indent=2)) diff --git a/tests/verify_adoption_drift.py b/tests/verify_adoption_drift.py new file mode 100644 index 0000000..6010be0 --- /dev/null +++ b/tests/verify_adoption_drift.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_adoption_drift" +SCRIPT = ROOT / "scripts" / "render_adoption_drift_report.py" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stderr": proc.stderr, + } + + +def record(skill_dir: Path, *args: str) -> dict: + return run([sys.executable, str(SCRIPT), str(skill_dir), *args]) + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + TMP.mkdir(parents=True, exist_ok=True) + + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "telemetry-demo", + "--description", + "Turn repeated operational signals into a reusable telemetry demo skill.", + "--output-dir", + str(TMP), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + skill_dir = TMP / "telemetry-demo" + + assert record( + skill_dir, + "--record-event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--timestamp", + "2026-06-13T10:00:00Z", + )["ok"] + assert record( + skill_dir, + "--record-event", + "skill_activation", + "--activation-type", + "implicit", + "--outcome", + "edited", + "--timestamp", + "2026-06-13T10:01:00Z", + )["ok"] + assert record( + skill_dir, + "--record-event", + "skill_activation", + "--activation-type", + "implicit", + "--outcome", + "missed", + "--failure-type", + "under_trigger", + "--timestamp", + "2026-06-13T10:02:00Z", + )["ok"] + assert record( + skill_dir, + "--record-event", + "skill_output", + "--outcome", + "rejected", + "--failure-type", + "bad_output", + "--timestamp", + "2026-06-13T10:03:00Z", + )["ok"] + final = record( + skill_dir, + "--record-event", + "script_run", + "--outcome", + "failed", + "--failure-type", + "script_error", + "--timestamp", + "2026-06-13T10:04:00Z", + ) + assert final["ok"], final + summary = final["payload"]["summary"] + assert summary["event_count"] == 5, summary + assert summary["adoption_rate"] == 40.0, summary + assert summary["missed_trigger_count"] == 2, summary + assert summary["bad_output_count"] == 1, summary + assert summary["script_error_count"] == 1, summary + assert summary["risk_band"] == "high", summary + assert final["payload"]["privacy_contract"]["raw_content_allowed"] is False, final + assert final["payload"]["next_iteration_candidates"], final + markdown = (skill_dir / "reports" / "adoption_drift_report.md").read_text(encoding="utf-8") + assert "metadata-only telemetry" in markdown, markdown + assert "Raw user prompts" in markdown, markdown + + unsafe_events = TMP / "unsafe_events.jsonl" + unsafe_events.write_text( + json.dumps( + { + "event": "skill_activation", + "outcome": "accepted", + "prompt": "raw prompt must not be stored", + } + ) + + "\n", + encoding="utf-8", + ) + unsafe = run( + [ + sys.executable, + str(SCRIPT), + str(skill_dir), + "--events-jsonl", + str(unsafe_events), + "--output-json", + str(TMP / "unsafe.json"), + "--output-md", + str(TMP / "unsafe.md"), + ] + ) + assert unsafe["returncode"] == 2, unsafe + assert not unsafe["payload"]["ok"], unsafe + assert "raw content fields" in unsafe["payload"]["failures"][0], unsafe + + root_events = ROOT / "reports" / "telemetry_events.jsonl" + original_root_events = root_events.read_text(encoding="utf-8") if root_events.exists() else None + root_events.write_text( + json.dumps( + { + "event": "skill_activation", + "skill": "yao-meta-skill", + "version": "1.1.0", + "activation_type": "explicit", + "outcome": "accepted", + "failure_type": "none", + "timestamp": "2026-06-13T10:05:00Z", + } + ) + + "\n", + encoding="utf-8", + ) + try: + package_dir = TMP / "dist" + package = run( + [ + sys.executable, + str(ROOT / "scripts" / "cross_packager.py"), + str(ROOT), + "--platform", + "generic", + "--output-dir", + str(package_dir), + "--zip", + ] + ) + assert package["ok"], package + with zipfile.ZipFile(package_dir / "yao-meta-skill.zip") as archive: + names = archive.namelist() + assert "yao-meta-skill/reports/telemetry_events.jsonl" not in names, names + finally: + if original_root_events is not None: + root_events.write_text(original_root_events, encoding="utf-8") + elif root_events.exists(): + root_events.unlink() + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_compile_skill.py b/tests/verify_compile_skill.py new file mode 100644 index 0000000..be5d6c5 --- /dev/null +++ b/tests/verify_compile_skill.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "compile_skill.py" +TRUST_SCRIPT = ROOT / "scripts" / "trust_check.py" + + +def run(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_compile_skill" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + output_json = tmp_root / "compiled_targets.json" + output_md = tmp_root / "compiled_targets.md" + subprocess.run( + [sys.executable, str(TRUST_SCRIPT), str(ROOT)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + + proc = run( + str(ROOT), + "--target", + "openai", + "--target", + "claude", + "--target", + "generic", + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ) + payload = json.loads(proc.stdout) + assert payload["ok"], payload + assert payload["schema_version"] == "1.0", payload + assert payload["summary"]["target_count"] == 3, payload + assert payload["summary"]["pass_count"] == 3, payload + assert output_json.exists(), output_json + assert output_md.exists(), output_md + assert "Compiled Targets" in output_md.read_text(encoding="utf-8") + by_target = {item["target"]: item for item in payload["targets"]} + for target in ("openai", "claude", "generic"): + item = by_target[target] + assert item["compiler"]["name"] == "yao-skill-ir-compiler", item + assert item["compiler"]["source"] == "skill-ir", item + assert item["compiled_contract"]["name"] == "yao-meta-skill", item + assert item["compiled_contract"]["target"] == target, item + assert item["compiled_contract"]["trigger"]["description"], item + assert item["compiled_contract"]["resources"]["counts"]["references"] > 0, item + assert item["compiled_contract"]["eval_plan"]["counts"]["output"] > 0, item + assert item["compiled_contract"]["permissions"]["source"] == "reports/security_trust_report.json", item + assert "file_write" in item["compiled_contract"]["permissions"]["declared_capabilities"], item + assert item["compiled_contract"]["target_permission_contract"]["target"] == target, item + assert item["compiled_contract"]["target_permission_contract"]["representation"], item + assert item["permission_contract"]["help_smoke"]["failed_count"] == 0, item + assert item["target_permission_contract"]["target"] == target, item + assert item["target_permission_contract"]["capability_counts"]["file_write"] > 0, item + native = item["target_native_contract"] + assert native["target"] == target, item + assert native["native_surface"], item + assert native["activation"]["policy"], item + assert native["resources"]["strategy"], item + assert native["scripts"]["strategy"], item + assert native["permissions"]["enforcement"], item + assert native["review"]["artifacts"], item + assert item["compiled_contract"]["target_native_contract"]["target"] == target, item + assert item["target_transform"]["generated_files"], item + assert item["target_transform"]["metadata_mapping"], item + assert item["target_transform"]["permission_representation"], item + assert item["target_transform"]["native_surface"] == native["native_surface"], item + assert item["target_transform"]["activation_policy"] == native["activation"]["policy"], item + + unsupported = run(str(ROOT), "--target", "unsupported-target", "--output-json", str(tmp_root / "bad.json"), "--output-md", str(tmp_root / "bad.md"), check=False) + bad_payload = json.loads(unsupported.stdout) + assert unsupported.returncode == 2, unsupported.stdout + assert not bad_payload["ok"], bad_payload + assert bad_payload["summary"]["block_count"] == 1, bad_payload + assert bad_payload["failures"], bad_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_install_simulation.py b/tests/verify_install_simulation.py new file mode 100644 index 0000000..60c79de --- /dev/null +++ b/tests/verify_install_simulation.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_install_simulation" +PACKAGER = ROOT / "scripts" / "cross_packager.py" +SIMULATOR = ROOT / "scripts" / "simulate_install.py" +EXPECTATIONS = ROOT / "evals" / "packaging_expectations.json" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = {} + if proc.stdout.strip(): + payload = json.loads(proc.stdout) + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + "payload": payload, + } + + +def build_package(out_dir: Path) -> dict: + return run( + [ + sys.executable, + str(PACKAGER), + str(ROOT), + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--expectations", + str(EXPECTATIONS), + "--output-dir", + str(out_dir), + "--zip", + ] + ) + + +def simulate(package_dir: Path, output_json: Path, output_md: Path) -> dict: + return run( + [ + sys.executable, + str(SIMULATOR), + str(ROOT), + "--package-dir", + str(package_dir), + "--install-root", + str(TMP / "install-root"), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ] + ) + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + TMP.mkdir(parents=True, exist_ok=True) + + valid_dir = TMP / "dist" + build = build_package(valid_dir) + assert build["ok"], build + valid = simulate(valid_dir, TMP / "install_simulation.json", TMP / "install_simulation.md") + payload = valid["payload"] + assert valid["ok"], valid + assert payload["ok"], payload + assert payload["summary"]["archive_extracted"], payload + assert payload["summary"]["entrypoint_loaded"], payload + assert payload["summary"]["manifest_loaded"], payload + assert payload["summary"]["interface_loaded"], payload + assert payload["summary"]["adapter_count"] == 3, payload + assert not payload["failures"], payload + assert "Install Simulation" in (TMP / "install_simulation.md").read_text(encoding="utf-8") + + unsafe_dir = TMP / "unsafe-dist" + shutil.copytree(valid_dir, unsafe_dir) + with zipfile.ZipFile(unsafe_dir / "yao-meta-skill.zip", "a", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("../evil.txt", "bad") + unsafe = simulate(unsafe_dir, TMP / "unsafe.json", TMP / "unsafe.md") + unsafe_payload = unsafe["payload"] + assert unsafe["returncode"] == 2, unsafe + assert not unsafe_payload["ok"], unsafe_payload + assert any("Archive has no absolute or parent-traversal entries" in item for item in unsafe_payload["failures"]), unsafe_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_output_eval_lab.py b/tests/verify_output_eval_lab.py index 15c5d2e..bf7ebf5 100644 --- a/tests/verify_output_eval_lab.py +++ b/tests/verify_output_eval_lab.py @@ -14,6 +14,9 @@ def main() -> None: tmp_root.mkdir(parents=True, exist_ok=True) output_json = tmp_root / "output_quality_scorecard.json" output_md = tmp_root / "output_quality_scorecard.md" + blind_pack_json = tmp_root / "output_blind_review_pack.json" + blind_pack_md = tmp_root / "output_blind_review_pack.md" + blind_answer_key_json = tmp_root / "output_blind_answer_key.json" proc = subprocess.run( [ sys.executable, @@ -24,6 +27,12 @@ def main() -> None: str(output_json), "--output-md", str(output_md), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--blind-answer-key-json", + str(blind_answer_key_json), ], cwd=ROOT, capture_output=True, @@ -39,12 +48,30 @@ def main() -> None: assert payload["summary"]["with_skill_pass_rate"] > payload["summary"]["baseline_pass_rate"], payload assert payload["summary"]["delta"] > 0, payload assert payload["summary"]["gate_pass"], payload + assert payload["summary"]["blind_pair_count"] == 5, payload + assert payload["blind_review"]["pair_count"] == 5, payload assert output_json.exists(), output_json assert output_md.exists(), output_md + assert blind_pack_json.exists(), blind_pack_json + assert blind_pack_md.exists(), blind_pack_md + assert blind_answer_key_json.exists(), blind_answer_key_json + blind_pack = json.loads(blind_pack_json.read_text(encoding="utf-8")) + answer_key = json.loads(blind_answer_key_json.read_text(encoding="utf-8")) + assert blind_pack["summary"]["pair_count"] == 5, blind_pack + assert blind_pack["summary"]["answer_key_separate"], blind_pack + assert answer_key["summary"]["pair_count"] == 5, answer_key + assert "variant_a_role" not in json.dumps(blind_pack, ensure_ascii=False), blind_pack + assert "variant_b_role" not in json.dumps(blind_pack, ensure_ascii=False), blind_pack + assert "variant_a_role" in json.dumps(answer_key, ensure_ascii=False), answer_key + assert all(pair["review_instruction"].startswith("Pick A or B") for pair in blind_pack["pairs"]), blind_pack markdown = output_md.read_text(encoding="utf-8") assert "Output Quality Scorecard" in markdown, markdown[:400] assert "with-skill" in markdown, markdown[:600] + assert "Blind A/B pairs" in markdown, markdown[:800] assert "Failure Taxonomy" in markdown, markdown[:1200] + blind_markdown = blind_pack_md.read_text(encoding="utf-8") + assert "Output Blind A/B Review Pack" in blind_markdown, blind_markdown[:400] + assert "Answer key separate" in blind_markdown, blind_markdown[:600] invalid_cases = tmp_root / "invalid_cases.jsonl" invalid_cases.write_text(json.dumps({"id": "bad", "prompt": "x"}) + "\n", encoding="utf-8") @@ -58,6 +85,12 @@ def main() -> None: str(tmp_root / "invalid.json"), "--output-md", str(tmp_root / "invalid.md"), + "--blind-pack-json", + str(tmp_root / "invalid_blind.json"), + "--blind-pack-md", + str(tmp_root / "invalid_blind.md"), + "--blind-answer-key-json", + str(tmp_root / "invalid_answer_key.json"), ], cwd=ROOT, capture_output=True, diff --git a/tests/verify_output_execution_runs.py b/tests/verify_output_execution_runs.py new file mode 100644 index 0000000..a488185 --- /dev/null +++ b/tests/verify_output_execution_runs.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "run_output_execution.py" +LOCAL_RUNNER = ROOT / "scripts" / "local_output_eval_runner.py" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_output_execution" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + + recorded_json = tmp_root / "recorded.json" + recorded_md = tmp_root / "recorded.md" + recorded_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(recorded_json), + "--output-md", + str(recorded_md), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + recorded = json.loads(recorded_proc.stdout) + assert recorded["ok"], recorded + assert recorded["summary"]["case_count"] == 5, recorded + assert recorded["summary"]["variant_run_count"] == 10, recorded + assert recorded["summary"]["recorded_fixture_count"] == 10, recorded + assert recorded["summary"]["command_executed_count"] == 0, recorded + assert recorded["summary"]["model_executed_count"] == 0, recorded + assert recorded["summary"]["token_estimated_count"] == 10, recorded + assert recorded["summary"]["with_skill_pass_rate"] > recorded["summary"]["baseline_pass_rate"], recorded + assert "No model-executed runs are recorded yet" in recorded_md.read_text(encoding="utf-8"), recorded_md + + local_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(tmp_root / "local.json"), + "--output-md", + str(tmp_root / "local.md"), + "--runner-command", + json.dumps([sys.executable, str(LOCAL_RUNNER)]), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + local = json.loads(local_proc.stdout) + assert local["ok"], local + assert local["summary"]["variant_run_count"] == 10, local + assert local["summary"]["command_executed_count"] == 10, local + assert local["summary"]["recorded_fixture_count"] == 0, local + assert local["summary"]["model_executed_count"] == 0, local + assert local["summary"]["timing_observed_count"] == 10, local + assert local["summary"]["token_estimated_count"] == 10, local + assert all(item["provider"] == "local-output-eval-runner" for item in local["runs"]), local + local_md = (tmp_root / "local.md").read_text(encoding="utf-8") + assert "Command runner evidence is present" in local_md, local_md + assert "not provider-backed model evidence" in local_md, local_md + + runner = tmp_root / "runner.py" + runner.write_text( + "\n".join( + [ + "import json, sys, time", + "request = json.loads(sys.stdin.read())", + "time.sleep(0.001)", + "print(json.dumps({", + " 'output': request['fixture_output'],", + " 'execution_kind': 'model',", + " 'provider': 'local-fixture',", + " 'model': 'fixture-model',", + " 'usage': {'input_tokens': 11, 'output_tokens': 17, 'total_tokens': 28}", + "}))", + ] + ) + + "\n", + encoding="utf-8", + ) + command_json = json.dumps([sys.executable, str(runner)]) + command_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(tmp_root / "command.json"), + "--output-md", + str(tmp_root / "command.md"), + "--runner-command", + command_json, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + command = json.loads(command_proc.stdout) + assert command["ok"], command + assert command["summary"]["command_executed_count"] == 10, command + assert command["summary"]["model_executed_count"] == 10, command + assert command["summary"]["timing_observed_count"] == 10, command + assert command["summary"]["token_observed_count"] == 10, command + assert command["summary"]["token_estimated_count"] == 0, command + assert all(item["duration_ms"] is not None for item in command["runs"]), command + assert all(item["model"] == "fixture-model" for item in command["runs"]), command + + bad_runner = tmp_root / "bad_runner.py" + bad_runner.write_text("import sys\nsys.exit(3)\n", encoding="utf-8") + bad_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(tmp_root / "bad.json"), + "--output-md", + str(tmp_root / "bad.md"), + "--runner-command", + json.dumps([sys.executable, str(bad_runner)]), + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert bad_proc.returncode == 2, bad_proc.stdout + bad = json.loads(bad_proc.stdout) + assert not bad["ok"], bad + assert bad["summary"]["failure_count"] == 10, bad + assert bad["failures"], bad + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_output_review_adjudication.py b/tests/verify_output_review_adjudication.py new file mode 100644 index 0000000..93ac4fa --- /dev/null +++ b/tests/verify_output_review_adjudication.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +OUTPUT_EVAL = ROOT / "scripts" / "run_output_eval.py" +ADJUDICATOR = ROOT / "scripts" / "adjudicate_output_review.py" + + +def run(args: list[str], check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, *args], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_output_review_adjudication" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + + scorecard_json = tmp_root / "output_quality_scorecard.json" + scorecard_md = tmp_root / "output_quality_scorecard.md" + blind_pack_json = tmp_root / "output_blind_review_pack.json" + blind_pack_md = tmp_root / "output_blind_review_pack.md" + answer_key_json = tmp_root / "output_blind_answer_key.json" + + run( + [ + str(OUTPUT_EVAL), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(scorecard_json), + "--output-md", + str(scorecard_md), + "--blind-pack-json", + str(blind_pack_json), + "--blind-pack-md", + str(blind_pack_md), + "--blind-answer-key-json", + str(answer_key_json), + ] + ) + + pending_json = tmp_root / "pending_adjudication.json" + pending_md = tmp_root / "pending_adjudication.md" + pending_proc = run( + [ + str(ADJUDICATOR), + "--blind-pack", + str(blind_pack_json), + "--answer-key", + str(answer_key_json), + "--decisions", + str(tmp_root / "missing_decisions.json"), + "--output-json", + str(pending_json), + "--output-md", + str(pending_md), + ] + ) + pending_payload = json.loads(pending_proc.stdout) + assert pending_payload["ok"], pending_payload + assert pending_payload["summary"]["pair_count"] == 5, pending_payload + assert pending_payload["summary"]["judgment_count"] == 0, pending_payload + assert pending_payload["summary"]["pending_count"] == 5, pending_payload + assert pending_payload["summary"]["agreement_rate"] is None, pending_payload + assert pending_payload["summary"]["needs_review"], pending_payload + assert "No reviewer decisions recorded yet" in pending_md.read_text(encoding="utf-8"), pending_md + + template_path = tmp_root / "output_review_decisions.json" + template_proc = run( + [ + str(ADJUDICATOR), + "--blind-pack", + str(blind_pack_json), + "--answer-key", + str(answer_key_json), + "--decisions", + str(template_path), + "--output-json", + str(tmp_root / "template_adjudication.json"), + "--output-md", + str(tmp_root / "template_adjudication.md"), + "--write-template", + ] + ) + template_payload = json.loads(template_proc.stdout) + assert template_payload["ok"], template_payload + assert template_payload["template_written"], template_payload + template = json.loads(template_path.read_text(encoding="utf-8")) + assert len(template["decisions"]) == 5, template + assert all(item["winner_variant"] == "" for item in template["decisions"]), template + + answer_key = json.loads(answer_key_json.read_text(encoding="utf-8")) + decisions = { + "schema_version": "1.0", + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": item["case_id"], + "winner_variant": item["expected_winner_variant"], + "confidence": 0.9, + "reason": "Matches the rubric better.", + } + for item in answer_key["answers"] + ], + } + filled_path = tmp_root / "filled_decisions.json" + filled_path.write_text(json.dumps(decisions, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + filled_proc = run( + [ + str(ADJUDICATOR), + "--blind-pack", + str(blind_pack_json), + "--answer-key", + str(answer_key_json), + "--decisions", + str(filled_path), + "--output-json", + str(tmp_root / "filled_adjudication.json"), + "--output-md", + str(tmp_root / "filled_adjudication.md"), + ] + ) + filled_payload = json.loads(filled_proc.stdout) + assert filled_payload["ok"], filled_payload + assert filled_payload["summary"]["judgment_count"] == 5, filled_payload + assert filled_payload["summary"]["pending_count"] == 0, filled_payload + assert filled_payload["summary"]["agreement_count"] == 5, filled_payload + assert filled_payload["summary"]["agreement_rate"] == 100.0, filled_payload + assert all(item["status"] == "match" for item in filled_payload["pairs"]), filled_payload + + invalid = { + "schema_version": "1.0", + "reviewer": "Yao QA", + "reviewed_at": "2026-06-13", + "decisions": [ + { + "case_id": answer_key["answers"][0]["case_id"], + "winner_variant": "C", + "confidence": 0.9, + "reason": "Invalid variant should fail validation.", + } + ], + } + invalid_path = tmp_root / "invalid_decisions.json" + invalid_path.write_text(json.dumps(invalid, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + invalid_proc = run( + [ + str(ADJUDICATOR), + "--blind-pack", + str(blind_pack_json), + "--answer-key", + str(answer_key_json), + "--decisions", + str(invalid_path), + "--output-json", + str(tmp_root / "invalid_adjudication.json"), + "--output-md", + str(tmp_root / "invalid_adjudication.md"), + ], + check=False, + ) + assert invalid_proc.returncode == 2, invalid_proc.stdout + invalid_payload = json.loads(invalid_proc.stdout) + assert not invalid_payload["ok"], invalid_payload + assert invalid_payload["summary"]["invalid_decision_count"] == 1, invalid_payload + assert invalid_payload["failures"], invalid_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_package_verification.py b/tests/verify_package_verification.py new file mode 100644 index 0000000..4d51907 --- /dev/null +++ b/tests/verify_package_verification.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +import zipfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_package_verification" +PACKAGER = ROOT / "scripts" / "cross_packager.py" +VERIFIER = ROOT / "scripts" / "verify_package.py" +EXPECTATIONS = ROOT / "evals" / "packaging_expectations.json" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = {} + if proc.stdout.strip(): + payload = json.loads(proc.stdout) + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + "payload": payload, + } + + +def build_package(out_dir: Path) -> dict: + return run( + [ + sys.executable, + str(PACKAGER), + str(ROOT), + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--expectations", + str(EXPECTATIONS), + "--output-dir", + str(out_dir), + "--zip", + ] + ) + + +def verify_package(out_dir: Path, output_json: Path, output_md: Path) -> dict: + return run( + [ + sys.executable, + str(VERIFIER), + str(ROOT), + "--package-dir", + str(out_dir), + "--expectations", + str(EXPECTATIONS), + "--registry-json", + str(ROOT / "reports" / "registry_audit.json"), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--require-zip", + "--generated-at", + "2026-06-13", + ] + ) + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + TMP.mkdir(parents=True, exist_ok=True) + + valid_dir = TMP / "dist" + build = build_package(valid_dir) + assert build["ok"], build + valid = verify_package(valid_dir, TMP / "package_verification.json", TMP / "package_verification.md") + payload = valid["payload"] + assert valid["ok"], valid + assert payload["ok"], payload + assert payload["summary"]["target_count"] == 3, payload + assert payload["summary"]["adapter_count"] == 3, payload + assert payload["summary"]["archive_present"], payload + assert payload["summary"]["archive_sha256"], payload + assert not payload["failures"], payload + assert (TMP / "package_verification.md").exists(), TMP + + unsafe_dir = TMP / "unsafe-dist" + shutil.copytree(valid_dir, unsafe_dir) + with zipfile.ZipFile(unsafe_dir / "yao-meta-skill.zip", "a", compression=zipfile.ZIP_DEFLATED) as archive: + archive.writestr("../evil.txt", "bad") + unsafe = verify_package(unsafe_dir, TMP / "unsafe.json", TMP / "unsafe.md") + assert unsafe["returncode"] == 2, unsafe + unsafe_payload = unsafe["payload"] + assert not unsafe_payload["ok"], unsafe_payload + assert any("Archive has no absolute or parent-traversal entries" in item for item in unsafe_payload["failures"]), unsafe_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_registry_audit.py b/tests/verify_registry_audit.py new file mode 100644 index 0000000..a0f8d3d --- /dev/null +++ b/tests/verify_registry_audit.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "registry_audit.py" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_registry" + tmp_root.mkdir(parents=True, exist_ok=True) + output_json = tmp_root / "registry_audit.json" + output_md = tmp_root / "registry_audit.md" + registry_dir = tmp_root / "registry" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--registry-dir", + str(registry_dir), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + package = payload["package"] + assert payload["ok"], payload + assert payload["schema_version"] == "2.0", payload + assert package["name"] == "yao-meta-skill", package + assert package["version"] == "1.1.0", package + assert package["license"] == "MIT", package + assert package["checksums"]["package_sha256"], package + assert package["source"]["ir_schema_version"] == "2.0.0", package + assert "distribution" in package, package + if (ROOT / "reports" / "package_verification.json").exists(): + assert package["distribution"]["archive_verified"], package + assert package["checksums"]["archive_sha256"], package + if (ROOT / "reports" / "install_simulation.json").exists(): + assert package["distribution"]["install_simulated"], package + assert package["artifacts"]["install_simulation"].endswith("reports/install_simulation.md"), package + if (ROOT / "reports" / "adoption_drift_report.json").exists(): + assert package["artifacts"]["adoption_drift"].endswith("reports/adoption_drift_report.md"), package + if (ROOT / "reports" / "review_waivers.json").exists(): + assert package["artifacts"]["review_waivers"].endswith("reports/review_waivers.md"), package + if (ROOT / "reports" / "compiled_targets.json").exists(): + assert package["artifacts"]["compiled_targets"].endswith("reports/compiled_targets.md"), package + assert package["compatibility"]["openai"] == "pass", package + assert package["compatibility"]["claude"] == "pass", package + assert package["compatibility"]["generic"] == "pass", package + assert (registry_dir / "index.json").exists(), registry_dir + assert (registry_dir / "packages" / "yao-meta-skill.json").exists(), registry_dir + assert output_json.exists(), output_json + assert output_md.exists(), output_md + assert "Registry Audit" in output_md.read_text(encoding="utf-8") + + broken = tmp_root / "broken-registry-skill" + (broken / "reports").mkdir(parents=True, exist_ok=True) + (broken / "agents").mkdir(parents=True, exist_ok=True) + (broken / "SKILL.md").write_text( + "---\nname: broken-registry-skill\ndescription: Broken registry fixture.\n---\n\n# Broken\n", + encoding="utf-8", + ) + (broken / "manifest.json").write_text(json.dumps({"name": "broken-registry-skill"}, indent=2), encoding="utf-8") + (broken / "agents" / "interface.yaml").write_text( + "interface:\n display_name: Broken\n short_description: Broken\n default_prompt: Broken\ncompatibility:\n adapter_targets: [generic]\n trust:\n source_tier: local\n", + encoding="utf-8", + ) + broken_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(broken), + "--registry-dir", + str(tmp_root / "broken-registry"), + "--output-json", + str(tmp_root / "broken.json"), + "--output-md", + str(tmp_root / "broken.md"), + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert broken_proc.returncode == 2, broken_proc.stdout + broken_payload = json.loads(broken_proc.stdout) + assert not broken_payload["ok"], broken_payload + assert any("version" in item for item in broken_payload["failures"]), broken_payload + assert any("checksum" in item for item in broken_payload["failures"]), broken_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_review_annotations.py b/tests/verify_review_annotations.py new file mode 100644 index 0000000..f3b48fa --- /dev/null +++ b/tests/verify_review_annotations.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_review_annotations.py" +STUDIO = ROOT / "scripts" / "render_review_studio.py" + + +def run(*args: str, check: bool = True) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_review_annotations" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + + empty_json = tmp_root / "empty.json" + empty_md = tmp_root / "empty.md" + empty_proc = run(str(ROOT), "--output-json", str(empty_json), "--output-md", str(empty_md)) + empty_payload = json.loads(empty_proc.stdout) + assert empty_payload["ok"], empty_payload + assert empty_payload["summary"]["annotation_count"] == 0, empty_payload + assert empty_payload["summary"]["open_blocker_count"] == 0, empty_payload + assert "No reviewer annotations recorded yet." in empty_md.read_text(encoding="utf-8"), empty_md + + source_json = tmp_root / "review_annotations_input.json" + template_proc = run( + str(ROOT), + "--annotations-json", + str(source_json), + "--output-json", + str(tmp_root / "template.json"), + "--output-md", + str(tmp_root / "template.md"), + "--write-template", + ) + template_payload = json.loads(template_proc.stdout) + assert template_payload["ok"], template_payload + assert template_payload["template_written"], template_payload + assert source_json.exists(), source_json + + warning_json = tmp_root / "warning.json" + warning_md = tmp_root / "warning.md" + add_proc = run( + str(ROOT), + "--annotations-json", + str(source_json), + "--output-json", + str(warning_json), + "--output-md", + str(warning_md), + "--add-annotation", + "--annotation-id", + "ann-output-1", + "--gate-key", + "output-lab", + "--target-path", + "reports/output_quality_scorecard.md", + "--line", + "1", + "--severity", + "warning", + "--reviewer", + "Yao QA", + "--created-at", + "2026-06-13", + "--body", + "Clarify whether the output evidence is recorded-fixture or model-executed.", + "--suggested-action", + "Open reports/output_execution_runs.md before release wording.", + ) + warning_payload = json.loads(add_proc.stdout) + assert warning_payload["ok"], warning_payload + assert warning_payload["summary"]["annotation_count"] == 1, warning_payload + assert warning_payload["summary"]["open_warning_count"] == 1, warning_payload + annotation = warning_payload["annotations"][0] + assert annotation["target_exists"], annotation + assert annotation["source_excerpt"], annotation + assert "ann-output-1" in warning_md.read_text(encoding="utf-8"), warning_md + + blocker_source = tmp_root / "blocker_input.json" + blocker_source.write_text( + json.dumps( + { + "schema_version": "1.0", + "annotations": [ + { + "id": "ann-release-blocker", + "gate_key": "release-notes", + "target_path": "reports/skill-os-2-review.md", + "line": 1, + "severity": "blocker", + "status": "open", + "reviewer": "Yao QA", + "created_at": "2026-06-13", + "body": "Release narrative needs one reviewer pass before publish.", + } + ], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + blocker_json = ROOT / "reports" / "review_annotations.json" + blocker_md = ROOT / "reports" / "review_annotations.md" + original_json = blocker_json.read_text(encoding="utf-8") if blocker_json.exists() else None + original_md = blocker_md.read_text(encoding="utf-8") if blocker_md.exists() else None + try: + blocker_proc = run( + str(ROOT), + "--annotations-json", + str(blocker_source), + "--output-json", + str(blocker_json), + "--output-md", + str(blocker_md), + ) + blocker_payload = json.loads(blocker_proc.stdout) + assert blocker_payload["ok"], blocker_payload + assert blocker_payload["summary"]["open_blocker_count"] == 1, blocker_payload + + studio_json = tmp_root / "review-studio-blocked.json" + studio_html = tmp_root / "review-studio-blocked.html" + studio_proc = subprocess.run( + [ + sys.executable, + str(STUDIO), + str(ROOT), + "--output-html", + str(studio_html), + "--output-json", + str(studio_json), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + studio_payload = json.loads(studio_proc.stdout) + assert studio_payload["summary"]["decision"] == "blocked", studio_payload + assert studio_payload["summary"]["open_annotation_blocker_count"] == 1, studio_payload + html = studio_html.read_text(encoding="utf-8") + assert "审查批注" in html, html[:4000] + assert "ann-release-blocker" in html, html + finally: + if original_json is None: + blocker_json.unlink(missing_ok=True) + else: + blocker_json.write_text(original_json, encoding="utf-8") + if original_md is None: + blocker_md.unlink(missing_ok=True) + else: + blocker_md.write_text(original_md, encoding="utf-8") + + invalid_proc = run( + str(ROOT), + "--annotations-json", + str(tmp_root / "invalid_input.json"), + "--output-json", + str(tmp_root / "invalid.json"), + "--output-md", + str(tmp_root / "invalid.md"), + "--add-annotation", + "--gate-key", + "output-lab", + "--target-path", + "../outside.md", + "--body", + "This path must be rejected.", + check=False, + ) + assert invalid_proc.returncode == 2, invalid_proc.stdout + invalid_payload = json.loads(invalid_proc.stdout) + assert not invalid_payload["ok"], invalid_payload + assert "escapes skill directory" in "\n".join(invalid_payload["failures"]), invalid_payload + assert not (tmp_root / "invalid_input.json").exists(), invalid_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index f4473f1..591fb6c 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import json +import shutil import subprocess import sys from pathlib import Path @@ -7,12 +8,203 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_review_studio.py" +sys.path.insert(0, str(ROOT / "scripts")) +import render_review_studio as review_studio # noqa: E402 def main() -> None: tmp_root = ROOT / "tests" / "tmp_review_studio" + if tmp_root.exists(): + shutil.rmtree(tmp_root) tmp_root.mkdir(parents=True, exist_ok=True) subprocess.run([sys.executable, str(ROOT / "scripts" / "run_output_eval.py")], cwd=ROOT, check=True, capture_output=True, text=True) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "run_output_execution.py"), + "--runner-command", + json.dumps(["python3", "scripts/local_output_eval_runner.py"]), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "adjudicate_output_review.py")], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "compile_skill.py"), str(ROOT), "--generated-at", "2026-06-13"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + package_dir = tmp_root / "dist" + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "cross_packager.py"), + str(ROOT), + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--expectations", + str(ROOT / "evals" / "packaging_expectations.json"), + "--output-dir", + str(package_dir), + "--zip", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "simulate_install.py"), + str(ROOT), + "--package-dir", + str(package_dir), + "--install-root", + str(tmp_root / "install-root"), + "--output-json", + str(ROOT / "reports" / "install_simulation.json"), + "--output-md", + str(ROOT / "reports" / "install_simulation.md"), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "registry_audit.py"), str(ROOT), "--generated-at", "2026-06-13"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "render_intent_confidence.py"), str(ROOT)], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "build_skill_atlas.py"), + "--workspace-root", + str(ROOT), + "--output-dir", + str(ROOT / "skill_atlas"), + "--report-html", + str(ROOT / "reports" / "skill_atlas.html"), + "--report-json", + str(ROOT / "reports" / "skill_atlas.json"), + "--today", + "2026-06-13", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "upgrade_check.py"), + str(ROOT), + "--previous-package-json", + str(ROOT / "registry" / "examples" / "yao-meta-skill-1.0.0.json"), + "--current-package-json", + str(ROOT / "reports" / "registry_audit.json"), + "--output-json", + str(tmp_root / "upgrade_check.json"), + "--output-md", + str(tmp_root / "upgrade_check.md"), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + (ROOT / "reports" / "upgrade_check.json").write_text( + (tmp_root / "upgrade_check.json").read_text(encoding="utf-8"), + encoding="utf-8", + ) + (ROOT / "reports" / "upgrade_check.md").write_text( + (tmp_root / "upgrade_check.md").read_text(encoding="utf-8"), + encoding="utf-8", + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_adoption_drift_report.py"), + str(ROOT), + "--events-jsonl", + str(tmp_root / "telemetry_events.jsonl"), + "--record-event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--timestamp", + "2026-06-13T10:00:00Z", + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [sys.executable, str(ROOT / "scripts" / "render_review_waivers.py"), str(ROOT), "--generated-at", "2026-06-13"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_review_annotations.py"), + str(ROOT), + "--annotations-json", + str(tmp_root / "empty_review_annotations_input.json"), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "probe_runtime_permissions.py"), + str(ROOT), + "--package-dir", + str(package_dir), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) output_html = tmp_root / "review-studio.html" output_json = tmp_root / "review-studio.json" @@ -34,20 +226,135 @@ def main() -> None: payload = json.loads(proc.stdout) assert payload["ok"], payload assert payload["schema_version"] == "2.0", payload - assert payload["summary"]["gate_count"] == 8, payload - assert payload["summary"]["world_class_score"] > 0, payload + assert payload["summary"]["decision"] == "ready", payload + assert payload["summary"]["gate_count"] == 13, payload + assert payload["summary"]["world_class_score"] == 100, payload + assert payload["summary"]["warning_count"] == 0, payload + assert payload["summary"]["blocker_count"] == 0, payload + assert payload["summary"]["action_count"] == 0, payload + assert payload["summary"]["annotation_count"] == 0, payload + assert payload["summary"]["open_annotation_blocker_count"] == 0, payload + assert payload["summary"]["action_count"] == payload["summary"]["warning_count"] + payload["summary"]["blocker_count"], payload + assert payload["review_actions"] == [], payload gate_keys = {item["key"] for item in payload["gates"]} - assert {"intent-canvas", "trigger-lab", "output-lab", "runtime-matrix", "trust-report", "skill-atlas", "release-notes"} <= gate_keys, payload + assert {"intent-canvas", "trigger-lab", "output-lab", "runtime-matrix", "trust-report", "permission-gates", "permission-runtime", "skill-atlas", "operations-loop", "review-waivers", "registry-audit", "release-notes"} <= gate_keys, payload output_gate = next(item for item in payload["gates"] if item["key"] == "output-lab") assert output_gate["status"] == "pass", output_gate assert "5/5 cases" in output_gate["detail"], output_gate assert "file-backed 1" in output_gate["detail"], output_gate + assert "blind A/B 5" in output_gate["detail"], output_gate + assert "exec 10" in output_gate["detail"], output_gate + assert "model 0" in output_gate["detail"], output_gate + assert "reviewed 0/5" in output_gate["detail"], output_gate + release_gate = next(item for item in payload["gates"] if item["key"] == "release-notes") + assert "upgrade minor declared / minor recommended" in release_gate["detail"], release_gate + assert "reports/upgrade_check.json" in release_gate["evidence"], release_gate + registry_gate = next(item for item in payload["gates"] if item["key"] == "registry-audit") + assert "install pass" in registry_gate["detail"], registry_gate + assert "reports/install_simulation.json" in registry_gate["evidence"], registry_gate + trust_gate = next(item for item in payload["gates"] if item["key"] == "trust-report") + assert trust_gate["status"] == "pass", trust_gate + assert "2 network-capable scripts" in trust_gate["detail"], trust_gate + assert "0 help smoke failures" in trust_gate["detail"], trust_gate + permission_gate = next(item for item in payload["gates"] if item["key"] == "permission-gates") + assert permission_gate["status"] == "pass", permission_gate + assert "permissions approved" in permission_gate["detail"], permission_gate + assert "gaps 0" in permission_gate["detail"], permission_gate + permission_runtime_gate = next(item for item in payload["gates"] if item["key"] == "permission-runtime") + assert permission_runtime_gate["status"] == "pass", permission_runtime_gate + assert "metadata fallback 3" in permission_runtime_gate["detail"], permission_runtime_gate + assert "native 0" in permission_runtime_gate["detail"], permission_runtime_gate + intent_gate = next(item for item in payload["gates"] if item["key"] == "intent-canvas") + assert intent_gate["status"] == "pass", intent_gate + assert "intent confidence 100/100" in intent_gate["detail"], intent_gate + atlas_gate = next(item for item in payload["gates"] if item["key"] == "skill-atlas") + assert atlas_gate["status"] == "pass", atlas_gate + assert "actionable route collisions" in atlas_gate["detail"], atlas_gate + operations_gate = next(item for item in payload["gates"] if item["key"] == "operations-loop") + assert operations_gate["status"] == "pass", operations_gate + assert "metadata events" in operations_gate["detail"], operations_gate + assert "risk low" in operations_gate["detail"], operations_gate + assert "reports/adoption_drift_report.json" in operations_gate["evidence"], operations_gate + waivers_gate = next(item for item in payload["gates"] if item["key"] == "review-waivers") + assert waivers_gate["status"] == "pass", waivers_gate + assert "cover current warnings" in waivers_gate["detail"], waivers_gate + assert "reports/review_waivers.json" in waivers_gate["evidence"], waivers_gate assert output_html.exists(), output_html assert output_json.exists(), output_json + full_payload = json.loads(output_json.read_text(encoding="utf-8")) + assert full_payload["evidence_paths"]["compiled_targets"] == "reports/compiled_targets.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["output_execution"] == "reports/output_execution_runs.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["output_blind_review"] == "reports/output_blind_review_pack.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["output_review_adjudication"] == "reports/output_review_adjudication.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["review_annotations"] == "reports/review_annotations.md", full_payload["evidence_paths"] + assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"] + assert full_payload["data"]["output_execution"]["summary"]["command_executed_count"] == 10, full_payload["data"]["output_execution"] + assert full_payload["data"]["output_execution"]["summary"]["recorded_fixture_count"] == 0, full_payload["data"]["output_execution"] + assert full_payload["data"]["output_execution"]["summary"]["model_executed_count"] == 0, full_payload["data"]["output_execution"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["pending_count"] == 5, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["review_annotations"]["summary"]["annotation_count"] == 0, full_payload["data"]["review_annotations"] + assert full_payload["data"]["compiled_targets"]["summary"]["target_count"] >= 4, full_payload["data"]["compiled_targets"] + assert full_payload["data"]["compiled_targets"]["summary"]["block_count"] == 0, full_payload["data"]["compiled_targets"] + assert full_payload["data"]["runtime_permissions"]["summary"]["metadata_fallback_count"] == 3, full_payload["data"]["runtime_permissions"] + assert full_payload["evidence_paths"]["runtime_permissions"] == "reports/runtime_permission_probes.md", full_payload["evidence_paths"] + action_keys = {item["gate_key"] for item in full_payload["review_actions"]} + assert action_keys == set(), full_payload["review_actions"] + assert full_payload["data"]["atlas"]["summary"]["actionable_route_collision_count"] == 0, full_payload["data"]["atlas"] + assert full_payload["data"]["atlas"]["summary"]["non_actionable_issue_count"] >= 1, full_payload["data"]["atlas"] + synthetic_actions = review_studio.build_review_actions( + [ + { + "key": "output-lab", + "label": "输出实验", + "status": "warn", + "detail": "synthetic missing output coverage", + "evidence": "reports/output_quality_scorecard.json", + "link": "reports/output_quality_scorecard.md", + } + ], + ROOT, + output_html, + ) + assert len(synthetic_actions) == 1, synthetic_actions + assert synthetic_actions[0]["source_refs"], synthetic_actions + assert {item["path"] for item in synthetic_actions[0]["source_refs"]} >= { + "evals/output/cases.jsonl", + "reports/output_quality_scorecard.md", + "reports/output_execution_runs.md", + "reports/output_blind_review_pack.md", + "reports/output_review_adjudication.md", + }, synthetic_actions + assert all(item["exists"] for item in synthetic_actions[0]["source_refs"]), synthetic_actions + assert all(isinstance(item["line"], int) and item["line"] >= 1 for item in synthetic_actions[0]["source_refs"]), synthetic_actions + synthetic_json = json.dumps(synthetic_actions, ensure_ascii=False) + assert str(ROOT) not in synthetic_json, synthetic_json + synthetic_html = review_studio.render_review_actions(synthetic_actions) + assert "source-ref-list" in synthetic_html, synthetic_html + assert "evals/output/cases.jsonl" in synthetic_html, synthetic_html html = output_html.read_text(encoding="utf-8") assert "Review Studio 2.0" in html, html[:400] assert "审查闸门" in html, html[:1200] + assert "修复动作" in html, html[:3000] + assert "当前没有 blocker 或 warning" in html, html[:9000] + assert "审查批注" in html, html[:9000] + assert "当前没有 reviewer 批注" in html, html[:9000] assert "输出实验" in html, html[:2000] + assert "执行证据" in html, html + assert "盲评包" in html, html[:5000] + assert "审定报告" in html, html + assert "注册审计" in html, html[:3000] + assert "包体验证" in html, html[:5000] + assert "Upgrade" in html, html[:6000] + assert "Compiler" in html, html[:6000] + assert "目标编译" in html, html[:9000] + assert "reports/compiled_targets.md" in output_json.read_text(encoding="utf-8"), output_json + assert "Install" in html, html[:6500] + assert "运营回路" in html, html[:7600] + assert "人工批准" in html, html[:8200] + assert "权限批准" in html, html[:9000] + assert "权限探针" in html, html[:9500] + assert "reports/review_waivers.md" in output_json.read_text(encoding="utf-8"), output_json + assert "upgrade minor declared / minor recommended" in html, html[:8000] assert str(ROOT) not in output_json.read_text(encoding="utf-8"), output_json print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_review_waivers.py b/tests/verify_review_waivers.py new file mode 100644 index 0000000..25e1aee --- /dev/null +++ b/tests/verify_review_waivers.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_review_waivers.py" + + +def run(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + cwd=ROOT, + capture_output=True, + text=True, + check=check, + ) + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_review_waivers" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + skill = tmp_root / "waiver-demo" + (skill / "reports").mkdir(parents=True, exist_ok=True) + (skill / "SKILL.md").write_text("---\nname: waiver-demo\ndescription: Waiver demo.\n---\n\n# Demo\n", encoding="utf-8") + + empty = run(str(skill), "--generated-at", "2026-06-13") + empty_payload = json.loads(empty.stdout) + assert empty_payload["ok"], empty_payload + assert empty_payload["summary"]["waiver_count"] == 0, empty_payload + assert (skill / "reports" / "review_waivers.json").exists(), skill + assert (skill / "reports" / "review_waivers.md").exists(), skill + + added = run( + str(skill), + "--generated-at", + "2026-06-13", + "--add-waiver", + "--gate-key", + "trust-report", + "--reviewer", + "Yao Team", + "--reason", + "Network-capable scripts are reviewed and bounded for this release.", + "--expires-at", + "2026-09-30", + ) + added_payload = json.loads(added.stdout) + assert added_payload["ok"], added_payload + assert added_payload["summary"]["active_count"] == 1, added_payload + assert added_payload["summary"]["covered_gate_keys"] == ["trust-report"], added_payload + assert "trust-report" in (skill / "reports" / "review_waivers.md").read_text(encoding="utf-8") + + expired_source = tmp_root / "expired.json" + expired_source.write_text( + json.dumps( + { + "waivers": [ + { + "gate_key": "operations-loop", + "decision": "temporary-exception", + "reviewer": "Yao Team", + "reason": "No local telemetry sample exists yet, accepted only for historical demo review.", + "created_at": "2026-01-01", + "expires_at": "2026-01-31", + } + ] + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + expired = run(str(skill), "--waivers-json", str(expired_source), "--output-json", str(tmp_root / "expired_report.json"), "--output-md", str(tmp_root / "expired_report.md"), "--generated-at", "2026-06-13") + expired_payload = json.loads(expired.stdout) + assert expired_payload["ok"], expired_payload + assert expired_payload["summary"]["expired_count"] == 1, expired_payload + assert expired_payload["summary"]["active_count"] == 0, expired_payload + assert expired_payload["warnings"], expired_payload + + invalid_source = tmp_root / "invalid.json" + invalid_source.write_text( + json.dumps({"waivers": [{"gate_key": "trust-report", "reason": "too short"}]}, ensure_ascii=False), + encoding="utf-8", + ) + invalid = run(str(skill), "--waivers-json", str(invalid_source), "--output-json", str(tmp_root / "invalid_report.json"), "--output-md", str(tmp_root / "invalid_report.md"), "--generated-at", "2026-06-13", check=False) + invalid_payload = json.loads(invalid.stdout) + assert invalid.returncode == 2, invalid.stdout + assert not invalid_payload["ok"], invalid_payload + assert invalid_payload["summary"]["invalid_count"] == 1, invalid_payload + assert invalid_payload["failures"], invalid_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_runtime_permission_probes.py b/tests/verify_runtime_permission_probes.py new file mode 100644 index 0000000..f5ef9ab --- /dev/null +++ b/tests/verify_runtime_permission_probes.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +TMP = ROOT / "tests" / "tmp_runtime_permission" +SCRIPT = ROOT / "scripts" / "probe_runtime_permissions.py" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = json.loads(proc.stdout) if proc.stdout.strip() else {} + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "payload": payload, + "stderr": proc.stderr, + } + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + TMP.mkdir(parents=True, exist_ok=True) + package_dir = TMP / "dist" + + subprocess.run([sys.executable, str(ROOT / "scripts" / "trust_check.py"), str(ROOT)], cwd=ROOT, check=True, capture_output=True, text=True) + package = run( + [ + sys.executable, + str(ROOT / "scripts" / "cross_packager.py"), + str(ROOT), + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--expectations", + str(ROOT / "evals" / "packaging_expectations.json"), + "--output-dir", + str(package_dir), + "--zip", + ] + ) + assert package["ok"], package + + probe = run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--package-dir", + str(package_dir), + "--output-json", + str(TMP / "runtime_permission_probes.json"), + "--output-md", + str(TMP / "runtime_permission_probes.md"), + ] + ) + assert probe["ok"], probe + payload = probe["payload"] + summary = payload["summary"] + assert summary["target_count"] == 3, summary + assert summary["pass_count"] == 3, summary + assert summary["fail_count"] == 0, summary + assert summary["native_enforcement_count"] == 0, summary + assert summary["metadata_fallback_count"] == 3, summary + assert summary["residual_risk_count"] == 3, summary + assert payload["expected_capabilities"] == ["file_write", "network", "subprocess"], payload + assert {item["assurance"] for item in payload["targets"]} == {"metadata-fallback-explicit"}, payload["targets"] + assert (TMP / "runtime_permission_probes.md").exists(), TMP + + bad_dir = TMP / "bad-dist" + shutil.copytree(package_dir, bad_dir) + openai_adapter = bad_dir / "targets" / "openai" / "adapter.json" + bad_payload = json.loads(openai_adapter.read_text(encoding="utf-8")) + bad_payload["target_permission_contract"].pop("operator_note", None) + openai_adapter.write_text(json.dumps(bad_payload, ensure_ascii=False, indent=2), encoding="utf-8") + bad_probe = run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--package-dir", + str(bad_dir), + "--output-json", + str(TMP / "bad_runtime_permission_probes.json"), + "--output-md", + str(TMP / "bad_runtime_permission_probes.md"), + ] + ) + assert bad_probe["returncode"] == 2, bad_probe + assert not bad_probe["payload"]["ok"], bad_probe + assert any("operator_note" in item for item in bad_probe["payload"]["failures"]), bad_probe["payload"]["failures"] + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_atlas.py b/tests/verify_skill_atlas.py index 42fadf0..e2381df 100644 --- a/tests/verify_skill_atlas.py +++ b/tests/verify_skill_atlas.py @@ -94,6 +94,27 @@ def main() -> None: "- no_route: repeated pricing approval prompts are missed by the current library.\n", encoding="utf-8", ) + policy_dir = tmp_root / "skill_atlas" + policy_dir.mkdir() + (policy_dir / "policy.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "scope_rules": [ + { + "path_prefix": "beta-release-skill", + "scope": "example", + "actionable": False, + "reason": "fixture used to verify scoped portfolio warnings", + } + ], + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) output_dir = tmp_root / "atlas" report_html = tmp_root / "skill_atlas.html" @@ -121,7 +142,9 @@ def main() -> None: payload = json.loads(proc.stdout) summary = payload["summary"] assert summary["skill_count"] == 3, summary + assert summary["actionable_skill_count"] == 2, summary assert summary["route_collision_count"] >= 1, summary + assert summary["actionable_route_collision_count"] <= summary["route_collision_count"], summary assert summary["owner_gap_count"] >= 1, summary assert summary["stale_count"] >= 1, summary assert summary["shared_resource_count"] >= 1, summary @@ -135,9 +158,11 @@ def main() -> None: html = report_html.read_text(encoding="utf-8") assert "Skill Atlas" in html, html[:1000] assert "Route Collisions" in html, html[:3000] + assert "Actionable Issues" in html, html[:3000] assert "No-Route Opportunities" in html, html[:3000] catalog = json.loads((output_dir / "catalog.json").read_text(encoding="utf-8")) assert catalog["summary"]["skill_count"] == 3, catalog + assert payload["scope_policy"]["present"], payload["scope_policy"] print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_skill_overview.py b/tests/verify_skill_overview.py index 811fe65..6c63efb 100644 --- a/tests/verify_skill_overview.py +++ b/tests/verify_skill_overview.py @@ -50,6 +50,8 @@ def main() -> None: assert (created / "reports" / "intent-confidence.json").exists(), created assert (created / "reports" / "skill-overview.html").exists(), created assert (created / "reports" / "skill-overview.json").exists(), created + assert (created / "reports" / "compiled_targets.md").exists(), created + assert (created / "reports" / "compiled_targets.json").exists(), created assert (created / "reports" / "reference-synthesis.md").exists(), created assert (created / "reports" / "reference-synthesis.json").exists(), created assert (created / "reports" / "artifact-design-profile.md").exists(), created @@ -60,6 +62,10 @@ def main() -> None: assert (created / "reports" / "system-model.json").exists(), created assert (created / "reports" / "iteration-directions.md").exists(), created assert (created / "reports" / "iteration-directions.json").exists(), created + assert (created / "reports" / "adoption_drift_report.md").exists(), created + assert (created / "reports" / "adoption_drift_report.json").exists(), created + assert (created / "reports" / "review_waivers.md").exists(), created + assert (created / "reports" / "review_waivers.json").exists(), created overview_json = json.loads((created / "reports" / "skill-overview.json").read_text(encoding="utf-8")) directions_json = json.loads((created / "reports" / "iteration-directions.json").read_text(encoding="utf-8")) @@ -79,16 +85,41 @@ def main() -> None: } assert expected_v2_keys.issubset(overview_json.keys()), overview_json.keys() assert "reports/skill-ir.json" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/compiled_targets.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/output_quality_scorecard.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/output_blind_review_pack.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/runtime_permission_probes.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/skill_atlas.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/registry_audit.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/package_verification.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/install_simulation.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/upgrade_check.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/adoption_drift_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/review_waivers.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/review-studio.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert overview_json["skill_ir"]["schema_version"] in {"", "2.0.0"}, overview_json.get("skill_ir") + assert overview_json["compiled_targets"]["summary"]["target_count"] >= 3, overview_json.get("compiled_targets") + assert overview_json["compiled_targets"]["summary"]["block_count"] == 0, overview_json.get("compiled_targets") assert "output_quality" in overview_json, overview_json.keys() + assert "output_execution" in overview_json, overview_json.keys() + assert overview_json["output_execution"]["summary"] == {}, overview_json["output_execution"] + assert "output_blind_review" in overview_json, overview_json.keys() + assert overview_json["output_blind_review"]["summary"] == {}, overview_json["output_blind_review"] + assert "output_review_adjudication" in overview_json, overview_json.keys() + assert overview_json["output_review_adjudication"]["summary"] == {}, overview_json["output_review_adjudication"] assert "runtime_conformance" in overview_json, overview_json.keys() + assert "runtime_permissions" in overview_json, overview_json.keys() + assert overview_json["runtime_permissions"]["summary"] == {}, overview_json["runtime_permissions"] assert "trust_security" in overview_json, overview_json.keys() assert "skill_atlas" in overview_json, overview_json.keys() + assert "registry_distribution" in overview_json, overview_json.keys() + assert "package_verification" in overview_json, overview_json.keys() + assert "install_simulation" in overview_json, overview_json.keys() + assert "upgrade_check" in overview_json, overview_json.keys() + assert "adoption_drift" in overview_json, overview_json.keys() + assert "review_waivers" in overview_json, overview_json.keys() assert [item["title"] for item in overview_json["iteration_roadmap"]["items"]] == [ item["title"] for item in directions_json["directions"] ], { @@ -141,6 +172,10 @@ def main() -> None: assert "执行流程" in report_html, report_html[:5000] assert "调用方式" in report_html, report_html[:5000] assert "证据不足" in report_html or "证据充分" in report_html, report_html[:8000] + assert "执行证据" in report_html, report_html + assert "尚未生成输出执行证据报告" in report_html, report_html + assert "盲评审定" in report_html, report_html + assert "尚未生成盲评审定报告" in report_html, report_html assert "原始说明可切换到英文查看；默认中文报告保留结论与结构说明。" not in report_html, report_html[:12000] assert "理解用户请求" in report_html, report_html[:5000] assert overview_json["logic_steps"][0] in report_html, overview_json.get("logic_steps") diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py index a9cec4e..ead57fc 100644 --- a/tests/verify_trust_check.py +++ b/tests/verify_trust_check.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import json +import shutil import subprocess import sys from pathlib import Path @@ -34,6 +35,7 @@ compatibility: def main() -> None: tmp_root = ROOT / "tests" / "tmp_trust" + shutil.rmtree(tmp_root, ignore_errors=True) tmp_root.mkdir(parents=True, exist_ok=True) output_json = tmp_root / "security_trust_report.json" output_md = tmp_root / "security_trust_report.md" @@ -55,10 +57,55 @@ def main() -> None: payload = json.loads(proc.stdout) assert payload["ok"], payload assert payload["summary"]["secret_findings"] == 0, payload + assert payload["summary"]["package_hash_scope"] == "source-contract-without-generated-reports", payload + assert payload["summary"]["package_hash_file_count"] == payload["summary"]["scanned_files"], payload assert payload["summary"]["package_sha256"], payload + assert payload["summary"]["internal_module_count"] >= 3, payload + assert payload["summary"]["network_script_count"] == 2, payload + assert payload["summary"]["network_policy_covered_count"] == payload["summary"]["network_script_count"], payload + assert payload["summary"]["network_policy_missing_count"] == 0, payload + assert payload["summary"]["permission_required_count"] >= 3, payload + assert payload["summary"]["permission_approved_count"] == payload["summary"]["permission_required_count"], payload + assert payload["summary"]["permission_missing_count"] == 0, payload + assert payload["summary"]["permission_invalid_count"] == 0, payload + assert payload["summary"]["permission_expired_count"] == 0, payload + assert payload["summary"]["help_smoke_checked_count"] > 0, payload + assert payload["summary"]["help_smoke_failed_count"] == 0, payload + assert payload["help_smoke"]["enabled"], payload["help_smoke"] + assert payload["help_smoke"]["checked_count"] == payload["help_smoke"]["passed_count"], payload["help_smoke"] + assert payload["help_smoke"]["failed_scripts"] == [], payload["help_smoke"] + assert payload["help_smoke"]["skipped_count"] == payload["summary"]["internal_module_count"], payload["help_smoke"] + assert payload["network_policy"]["present"], payload["network_policy"] + assert payload["network_policy"]["missing_scripts"] == [], payload["network_policy"] + assert payload["network_policy"]["mismatches"] == [], payload["network_policy"] + assert payload["permission_governance"]["present"], payload["permission_governance"] + assert {"network", "file_write", "subprocess"} <= set(payload["permission_governance"]["approved_capabilities"]), payload["permission_governance"] + assert payload["permission_governance"]["missing_capabilities"] == [], payload["permission_governance"] + assert payload["permission_governance"]["invalid_capabilities"] == [], payload["permission_governance"] + assert payload["permission_governance"]["expired_capabilities"] == [], payload["permission_governance"] + assert "scripts/check_update.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] + assert "scripts/github_benchmark_scan.py" in payload["network_policy"]["covered_scripts"], payload["network_policy"] + script_map = {item["path"]: item for item in payload["scripts"]} + for internal_module in [ + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + ]: + assert script_map[internal_module]["interface"] == "internal-module", script_map[internal_module] + assert script_map[internal_module]["interface_declared"], script_map[internal_module] + warning_text = "\n".join(payload["warnings"]) + assert "skill_report_charts.py" not in warning_text, payload["warnings"] + assert "skill_report_metrics.py" not in warning_text, payload["warnings"] + assert "skill_report_model.py" not in warning_text, payload["warnings"] + assert "render_context_reports.py" not in warning_text, payload["warnings"] + assert "render_social_preview.py" not in warning_text, payload["warnings"] + assert "Network-capable scripts require bounded host policy" not in warning_text, payload["warnings"] + assert "CLI help smoke failed" not in warning_text, payload["warnings"] + assert "Permission approvals" not in warning_text, payload["warnings"] assert output_json.exists(), output_json assert output_md.exists(), output_md assert "Security Trust Report" in output_md.read_text(encoding="utf-8") + assert "Package hash scope" in output_md.read_text(encoding="utf-8") secret_skill = tmp_root / "secret-skill" (secret_skill / "agents").mkdir(parents=True, exist_ok=True) @@ -84,6 +131,128 @@ def main() -> None: assert secret_payload["summary"]["secret_findings"] == 1, secret_payload assert secret_payload["secrets"][0]["type"] == "github_token", secret_payload + network_skill = tmp_root / "network-skill" + (network_skill / "agents").mkdir(parents=True, exist_ok=True) + (network_skill / "scripts").mkdir(parents=True, exist_ok=True) + (network_skill / "SKILL.md").write_text( + "---\nname: network-skill\ndescription: Network demo.\n---\n\n# Network\n", + encoding="utf-8", + ) + (network_skill / "agents" / "interface.yaml").write_text(INTERFACE, encoding="utf-8") + (network_skill / "scripts" / "probe.py").write_text( + "from urllib.request import urlopen\n\nurlopen('https://example.com/status')\n", + encoding="utf-8", + ) + missing_policy_proc = subprocess.run( + [sys.executable, str(SCRIPT), str(network_skill)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + missing_policy_payload = json.loads(missing_policy_proc.stdout) + assert missing_policy_payload["summary"]["network_script_count"] == 1, missing_policy_payload + assert missing_policy_payload["summary"]["network_policy_missing_count"] == 1, missing_policy_payload + assert any("Network-capable scripts require bounded host policy" in item for item in missing_policy_payload["warnings"]), missing_policy_payload + + (network_skill / "security").mkdir(parents=True, exist_ok=True) + (network_skill / "security" / "network_policy.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "scripts": { + "scripts/probe.py": { + "purpose": "Test policy coverage.", + "allowed_hosts": ["example.com"], + "allowed_path_prefixes": ["/"], + "requires_https": True, + "custom_url_policy": "not supported", + } + }, + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + covered_policy_proc = subprocess.run( + [sys.executable, str(SCRIPT), str(network_skill)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + covered_policy_payload = json.loads(covered_policy_proc.stdout) + assert covered_policy_payload["summary"]["network_policy_covered_count"] == 1, covered_policy_payload + assert covered_policy_payload["summary"]["network_policy_missing_count"] == 0, covered_policy_payload + assert not any("Network-capable scripts require bounded host policy" in item for item in covered_policy_payload["warnings"]), covered_policy_payload + assert covered_policy_payload["summary"]["permission_missing_count"] == 1, covered_policy_payload + assert covered_policy_payload["permission_governance"]["missing_capabilities"] == ["network"], covered_policy_payload + assert any("Permission approvals missing: network" in item for item in covered_policy_payload["warnings"]), covered_policy_payload + + (network_skill / "security" / "permission_policy.json").write_text( + json.dumps( + { + "schema_version": "1.0", + "capabilities": { + "network": { + "decision": "approved", + "reviewer": "Yao Team", + "scope": "Test network probe.", + "reason": "Network access is bounded to example.com for this isolated test fixture.", + "expires_at": "2026-09-30", + "target_enforcement": { + "openai": "metadata-only", + "claude": "adapter metadata", + "generic": "adapter metadata", + }, + } + }, + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + approved_policy_proc = subprocess.run( + [sys.executable, str(SCRIPT), str(network_skill)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + approved_policy_payload = json.loads(approved_policy_proc.stdout) + assert approved_policy_payload["summary"]["permission_approved_count"] == 1, approved_policy_payload + assert approved_policy_payload["summary"]["permission_missing_count"] == 0, approved_policy_payload + assert not any("Permission approvals missing" in item for item in approved_policy_payload["warnings"]), approved_policy_payload + + help_smoke_skill = tmp_root / "help-smoke-skill" + (help_smoke_skill / "agents").mkdir(parents=True, exist_ok=True) + (help_smoke_skill / "scripts").mkdir(parents=True, exist_ok=True) + (help_smoke_skill / "SKILL.md").write_text( + "---\nname: help-smoke-skill\ndescription: Help smoke demo.\n---\n\n# Help Smoke\n", + encoding="utf-8", + ) + (help_smoke_skill / "agents" / "interface.yaml").write_text(INTERFACE, encoding="utf-8") + (help_smoke_skill / "scripts" / "bad_help.py").write_text( + "import argparse\n\nraise RuntimeError('help import failed')\n", + encoding="utf-8", + ) + help_smoke_proc = subprocess.run( + [sys.executable, str(SCRIPT), str(help_smoke_skill)], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + help_smoke_payload = json.loads(help_smoke_proc.stdout) + assert help_smoke_payload["summary"]["help_smoke_checked_count"] == 1, help_smoke_payload + assert help_smoke_payload["summary"]["help_smoke_failed_count"] == 1, help_smoke_payload + assert help_smoke_payload["help_smoke"]["failed_scripts"] == ["scripts/bad_help.py"], help_smoke_payload + assert any("CLI help smoke failed: scripts/bad_help.py" in item for item in help_smoke_payload["warnings"]), help_smoke_payload + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_upgrade_check.py b/tests/verify_upgrade_check.py new file mode 100644 index 0000000..945f381 --- /dev/null +++ b/tests/verify_upgrade_check.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "upgrade_check.py" +PREVIOUS = ROOT / "registry" / "examples" / "yao-meta-skill-1.0.0.json" + + +def run(cmd: list[str]) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = {} + if proc.stdout.strip(): + payload = json.loads(proc.stdout) + return { + "ok": proc.returncode == 0, + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + "payload": payload, + } + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_upgrade_check" + tmp_root.mkdir(parents=True, exist_ok=True) + output_json = tmp_root / "upgrade_check.json" + output_md = tmp_root / "upgrade_check.md" + result = run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--previous-package-json", + str(PREVIOUS), + "--current-package-json", + str(ROOT / "reports" / "registry_audit.json"), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ] + ) + payload = result["payload"] + assert result["ok"], result + assert payload["ok"], payload + assert payload["summary"]["declared_bump"] == "minor", payload + assert payload["summary"]["recommended_bump"] == "minor", payload + assert "agent-skills-compatible" in payload["upgrade_diff"]["added_targets"], payload + assert output_json.exists(), output_json + assert "Upgrade Check" in output_md.read_text(encoding="utf-8"), output_md + + previous = json.loads(PREVIOUS.read_text(encoding="utf-8")) + current = json.loads((ROOT / "reports" / "registry_audit.json").read_text(encoding="utf-8"))["package"] + current["version"] = "1.1.0" + current["targets"] = ["openai", "claude"] + current["compatibility"] = {"openai": "pass", "claude": "fail"} + bad_current = tmp_root / "bad-current.json" + bad_current.write_text(json.dumps(current, ensure_ascii=False, indent=2), encoding="utf-8") + bad = run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--previous-package-json", + str(PREVIOUS), + "--current-package-json", + str(bad_current), + "--output-json", + str(tmp_root / "bad.json"), + "--output-md", + str(tmp_root / "bad.md"), + "--generated-at", + "2026-06-13", + ] + ) + bad_payload = bad["payload"] + assert bad["returncode"] == 2, bad + assert not bad_payload["ok"], bad_payload + assert bad_payload["summary"]["recommended_bump"] == "major", bad_payload + assert any("Version bump is insufficient" in item for item in bad_payload["failures"]), bad_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 9ff14b3..6ad15ed 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -55,7 +55,15 @@ def main() -> None: assert (created / "reports" / "prompt-quality-profile.md").exists(), created assert (created / "reports" / "system-model.md").exists(), created assert (created / "reports" / "skill-ir.json").exists(), created + assert (created / "reports" / "compiled_targets.md").exists(), created + assert (created / "reports" / "compiled_targets.json").exists(), created assert (created / "reports" / "iteration-directions.md").exists(), created + assert (created / "reports" / "adoption_drift_report.md").exists(), created + assert (created / "reports" / "adoption_drift_report.json").exists(), created + assert (created / "reports" / "review_waivers.md").exists(), created + assert (created / "reports" / "review_waivers.json").exists(), created + assert (created / "reports" / "review_annotations.md").exists(), created + assert (created / "reports" / "review_annotations.json").exists(), created assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created init_report_view = init_result["payload"]["report_view"] assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view @@ -64,7 +72,9 @@ def main() -> None: assert Path(init_report_view["review_studio"]).exists(), init_report_view assert "Skill 已创建完成" in init_report_view["message"], init_report_view assert "Review Studio 2.0" in init_report_view["message"], init_report_view - assert "概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view + assert "目标编译" in init_report_view["message"], init_report_view + assert "reports/compiled_targets.md" in init_report_view["message"], init_report_view + assert "概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view assert "默认使用中文简体" in init_report_view["message"], init_report_view assert "切换英文版" in init_report_view["message"], init_report_view init_skill_ir = init_result["payload"]["skill_ir"] @@ -102,6 +112,11 @@ def main() -> None: assert (quickstart_root / "reports" / "artifact-design-profile.md").exists(), quickstart_root assert (quickstart_root / "reports" / "prompt-quality-profile.md").exists(), quickstart_root assert (quickstart_root / "reports" / "system-model.md").exists(), quickstart_root + assert (quickstart_root / "reports" / "compiled_targets.md").exists(), quickstart_root + assert (quickstart_root / "reports" / "compiled_targets.json").exists(), quickstart_root + assert (quickstart_root / "reports" / "adoption_drift_report.md").exists(), quickstart_root + assert (quickstart_root / "reports" / "review_waivers.md").exists(), quickstart_root + assert (quickstart_root / "reports" / "review_annotations.md").exists(), quickstart_root assert quickstart_result["payload"]["archetype"] == "production", quickstart_result assert quickstart_result["payload"]["guidance"]["experience_note"], quickstart_result assert quickstart_result["payload"]["guidance"]["problem_diagnosis"]["candidates"], quickstart_result @@ -176,7 +191,67 @@ def main() -> None: review_studio_result = run("review-studio", str(created)) assert review_studio_result["ok"], review_studio_result assert review_studio_result["payload"]["artifacts"]["html"].endswith("reports/review-studio.html"), review_studio_result - assert review_studio_result["payload"]["summary"]["gate_count"] == 8, review_studio_result + assert review_studio_result["payload"]["summary"]["gate_count"] == 13, review_studio_result + + review_waivers_result = run( + "review-waivers", + str(created), + "--add-waiver", + "--gate-key", + "trust-report", + "--reviewer", + "Yao Team", + "--reason", + "Trust warning accepted for this CLI demo with bounded release follow-up.", + "--expires-at", + "2026-09-30", + "--generated-at", + "2026-06-13", + ) + assert review_waivers_result["ok"], review_waivers_result + assert review_waivers_result["payload"]["summary"]["active_count"] == 1, review_waivers_result + assert "trust-report" in review_waivers_result["payload"]["summary"]["covered_gate_keys"], review_waivers_result + + review_annotations_result = run( + "review-annotations", + str(created), + "--add-annotation", + "--annotation-id", + "ann-cli-trigger", + "--gate-key", + "trigger-lab", + "--target-path", + "SKILL.md", + "--line", + "1", + "--severity", + "note", + "--reviewer", + "Yao QA", + "--created-at", + "2026-06-13", + "--body", + "Check trigger wording before reuse.", + ) + assert review_annotations_result["ok"], review_annotations_result + assert review_annotations_result["payload"]["summary"]["annotation_count"] == 1, review_annotations_result + assert (created / "reports" / "review_annotations.md").exists(), review_annotations_result + + registry_result = run( + "registry-audit", + str(ROOT), + "--registry-dir", + str(tmp_root / "registry"), + "--output-json", + str(tmp_root / "registry_audit.json"), + "--output-md", + str(tmp_root / "registry_audit.md"), + "--generated-at", + "2026-06-13", + ) + assert registry_result["ok"], registry_result + assert registry_result["payload"]["package"]["name"] == "yao-meta-skill", registry_result + assert registry_result["payload"]["package"]["checksums"]["package_sha256"], registry_result reference_scan_result = run( "reference-scan", @@ -246,6 +321,12 @@ def main() -> None: assert created_skill_ir["schema_version"] == "2.0.0", created_skill_ir assert created_skill_ir["trigger_surface"]["description"], created_skill_ir + compile_result = run("compile-skill", str(created), "--target", "openai", "--target", "claude", "--target", "generic") + assert compile_result["ok"], compile_result + assert compile_result["payload"]["summary"]["target_count"] == 3, compile_result + assert compile_result["payload"]["summary"]["block_count"] == 0, compile_result + assert compile_result["payload"]["artifacts"]["markdown"].endswith("reports/compiled_targets.md"), compile_result + output_eval_result = run( "output-eval", "--cases", @@ -254,9 +335,50 @@ def main() -> None: str(created / "reports" / "output_quality_scorecard.json"), "--output-md", str(created / "reports" / "output_quality_scorecard.md"), + "--blind-pack-json", + str(created / "reports" / "output_blind_review_pack.json"), + "--blind-pack-md", + str(created / "reports" / "output_blind_review_pack.md"), + "--blind-answer-key-json", + str(created / "reports" / "output_blind_answer_key.json"), ) assert output_eval_result["ok"], output_eval_result assert output_eval_result["payload"]["summary"]["with_skill_pass_rate"] > output_eval_result["payload"]["summary"]["baseline_pass_rate"], output_eval_result + assert output_eval_result["payload"]["summary"]["blind_pair_count"] == 5, output_eval_result + assert (created / "reports" / "output_blind_review_pack.md").exists(), output_eval_result + assert (created / "reports" / "output_blind_answer_key.json").exists(), output_eval_result + + output_exec_result = run( + "output-exec", + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(created / "reports" / "output_execution_runs.json"), + "--output-md", + str(created / "reports" / "output_execution_runs.md"), + ) + assert output_exec_result["ok"], output_exec_result + assert output_exec_result["payload"]["summary"]["variant_run_count"] == 10, output_exec_result + assert output_exec_result["payload"]["summary"]["recorded_fixture_count"] == 10, output_exec_result + assert (created / "reports" / "output_execution_runs.md").exists(), output_exec_result + + output_review_result = run( + "output-review", + "--blind-pack", + str(created / "reports" / "output_blind_review_pack.json"), + "--answer-key", + str(created / "reports" / "output_blind_answer_key.json"), + "--decisions", + str(created / "reports" / "output_review_decisions.json"), + "--output-json", + str(created / "reports" / "output_review_adjudication.json"), + "--output-md", + str(created / "reports" / "output_review_adjudication.md"), + ) + assert output_review_result["ok"], output_review_result + assert output_review_result["payload"]["summary"]["judgment_count"] == 0, output_review_result + assert output_review_result["payload"]["summary"]["pending_count"] == 5, output_review_result + assert (created / "reports" / "output_review_adjudication.md").exists(), output_review_result conformance_result = run("conformance", str(created)) assert conformance_result["ok"], conformance_result @@ -300,6 +422,24 @@ def main() -> None: assert feedback_result["ok"], feedback_result assert feedback_result["payload"]["feedback"]["summary"]["count"] == 1, feedback_result + adoption_drift_result = run( + "adoption-drift", + str(created), + "--record-event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--timestamp", + "2026-06-13T10:00:00Z", + ) + assert adoption_drift_result["ok"], adoption_drift_result + assert adoption_drift_result["payload"]["summary"]["event_count"] == 1, adoption_drift_result + assert adoption_drift_result["payload"]["artifacts"]["markdown"].endswith( + "reports/adoption_drift_report.md" + ), adoption_drift_result + optimize_result = run("optimize-description", "--target", "root") assert optimize_result["ok"], optimize_result assert optimize_result["payload"]["winner"]["label"] == "Current", optimize_result @@ -330,11 +470,110 @@ def main() -> None: assert "portability_score" in report_result["payload"]["artifacts"], report_result assert "artifact_design_profile" in report_result["payload"]["artifacts"], report_result assert "prompt_quality_profile" in report_result["payload"]["artifacts"], report_result + assert "compiled_targets" in report_result["payload"]["artifacts"], report_result + assert "output_execution" in report_result["payload"]["artifacts"], report_result + assert "output_review_adjudication" in report_result["payload"]["artifacts"], report_result + assert "adoption_drift" in report_result["payload"]["artifacts"], report_result + assert "review_annotations" in report_result["payload"]["artifacts"], report_result + report_output_execution = json.loads((ROOT / "reports" / "output_execution_runs.json").read_text(encoding="utf-8")) + assert report_output_execution["summary"]["command_executed_count"] == 10, report_output_execution + assert report_output_execution["summary"]["recorded_fixture_count"] == 0, report_output_execution + assert report_output_execution["summary"]["model_executed_count"] == 0, report_output_execution package_dir = tmp_root / "dist" package_result = run("package", ".", "--platform", "generic", "--output-dir", str(package_dir)) assert package_result["ok"], package_result assert (package_dir / "targets" / "generic" / "adapter.json").exists(), package_dir + generic_adapter = json.loads((package_dir / "targets" / "generic" / "adapter.json").read_text(encoding="utf-8")) + assert generic_adapter["compiler"]["name"] == "yao-skill-ir-compiler", generic_adapter + assert generic_adapter["compiled_contract"]["target"] == "generic", generic_adapter + + package_zip_dir = tmp_root / "dist-zip" + package_zip_result = run( + "package", + ".", + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--expectations", + str(ROOT / "evals" / "packaging_expectations.json"), + "--output-dir", + str(package_zip_dir), + "--zip", + ) + assert package_zip_result["ok"], package_zip_result + package_verify_result = run( + "package-verify", + ".", + "--package-dir", + str(package_zip_dir), + "--expectations", + str(ROOT / "evals" / "packaging_expectations.json"), + "--registry-json", + str(ROOT / "reports" / "registry_audit.json"), + "--output-json", + str(tmp_root / "package_verification.json"), + "--output-md", + str(tmp_root / "package_verification.md"), + "--require-zip", + "--generated-at", + "2026-06-13", + ) + assert package_verify_result["ok"], package_verify_result + assert package_verify_result["payload"]["summary"]["adapter_count"] == 3, package_verify_result + assert package_verify_result["payload"]["summary"]["archive_sha256"], package_verify_result + + runtime_permissions_result = run( + "runtime-permissions", + ".", + "--package-dir", + str(package_zip_dir), + "--output-json", + str(tmp_root / "runtime_permission_probes.json"), + "--output-md", + str(tmp_root / "runtime_permission_probes.md"), + ) + assert runtime_permissions_result["ok"], runtime_permissions_result + assert runtime_permissions_result["payload"]["summary"]["metadata_fallback_count"] == 3, runtime_permissions_result + assert runtime_permissions_result["payload"]["summary"]["native_enforcement_count"] == 0, runtime_permissions_result + + install_simulate_result = run( + "install-simulate", + ".", + "--package-dir", + str(package_zip_dir), + "--install-root", + str(tmp_root / "install-root"), + "--output-json", + str(tmp_root / "install_simulation.json"), + "--output-md", + str(tmp_root / "install_simulation.md"), + "--generated-at", + "2026-06-13", + ) + assert install_simulate_result["ok"], install_simulate_result + assert install_simulate_result["payload"]["summary"]["archive_extracted"], install_simulate_result + assert install_simulate_result["payload"]["summary"]["adapter_count"] == 3, install_simulate_result + + upgrade_result = run( + "upgrade-check", + ".", + "--previous-package-json", + str(ROOT / "registry" / "examples" / "yao-meta-skill-1.0.0.json"), + "--current-package-json", + str(ROOT / "reports" / "registry_audit.json"), + "--output-json", + str(tmp_root / "upgrade_check.json"), + "--output-md", + str(tmp_root / "upgrade_check.md"), + "--generated-at", + "2026-06-13", + ) + assert upgrade_result["ok"], upgrade_result + assert upgrade_result["payload"]["summary"]["recommended_bump"] == "minor", upgrade_result update_result = run( "check-update",