diff --git a/Makefile b/Makefile index dd1cdc63..4095fab8 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -58,6 +58,9 @@ skill-atlas-check: review-viewer-check: $(PYTHON) tests/verify_review_viewer.py +review-studio-check: + $(PYTHON) tests/verify_review_studio.py + feedback-check: $(PYTHON) tests/verify_feedback.py @@ -151,11 +154,11 @@ sync-local-install: sync-active-install: $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_output_eval tests/tmp_conformance tests/tmp_trust tests/tmp_skill_atlas tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_output_eval tests/tmp_conformance tests/tmp_trust tests/tmp_skill_atlas tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index f1d66c68..2a446546 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a front-loaded intent dialogue with an intent confidence gate, so the system keeps clarifying when the true job, outputs, exclusions, or standards are still fuzzy - a silent-by-default GitHub benchmark scan plus reference synthesis that studies top public repositories and world-class pattern tracks, then surfaces only real conflicts or uncertainty to the user - a generated visual HTML overview for each newly initialized skill +- a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, and release evidence - a side-by-side HTML review studio for first-pass human review - an artifact design profile that defines visual direction, layout patterns, and quality gates for reports, tutorials, dashboards, screenshots, and review pages - a prompt quality profile that abstracts need modeling, RTF mapping, complexity, and quality checks into reviewer-visible evidence instead of bloating `SKILL.md` @@ -30,7 +31,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a lightweight feedback log that does not require a full promotion cycle - a baseline compare report for with-skill vs baseline review - a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits -- neutral source metadata plus client-specific adapters +- Skill IR as the platform-neutral semantic contract plus client-specific adapters - governance, promotion, and portability checks built into the default flow ## Architecture @@ -84,7 +85,7 @@ Weighted score formula: `sum(score / 10 * weight)`. 2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit. 3. Let `quickstart` clarify intent first, then run silent benchmark scan and reference synthesis; it only surfaces explicit questions when intent is still unclear or when there is a real design conflict. 4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode. -5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then inspect `reports/intent-dialogue.md`, `reports/intent-confidence.md`, `reports/reference-synthesis.md`, `reports/artifact-design-profile.md`, `reports/prompt-quality-profile.md`, `reports/system-model.md`, and `reports/iteration-directions.md` before adding more structure. +5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/review-studio.html` to inspect release blockers and evidence paths in one page before adding more structure. Or use the unified authoring CLI: @@ -96,6 +97,7 @@ python3 scripts/yao.py reference-scan my-skill \ --user-reference "A product or repo I admire::taste::Learn the clarity and operating standard.::Do not copy wording." \ --local-constraint "Current Library Naming::structure::Keep naming aligned with the local skill library.::Do not inherit private references." python3 scripts/yao.py review-viewer my-skill +python3 scripts/yao.py review-studio my-skill python3 scripts/yao.py artifact-design-profile my-skill python3 scripts/yao.py prompt-quality-profile my-skill python3 scripts/yao.py system-model my-skill @@ -223,7 +225,7 @@ The homepage panel below is generated from the current eval suite so the family- Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md) -- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks +- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks and carry IR provenance plus semantic parity metadata - portability score: `100/100` with neutral activation, execution, trust, and degradation metadata preserved across all exported targets - description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately - judge-backed blind eval: root, team frontend review, and governed incident command now pass an independent rubric judge on blind holdout prompts @@ -241,6 +243,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - promotion decisions are published in [reports/promotion_decisions.md](reports/promotion_decisions.md) - candidate lifecycle states are published in [reports/candidate_registry.md](reports/candidate_registry.md) - lightweight with-skill vs baseline comparison is published in [reports/baseline-compare.md](reports/baseline-compare.md) +- Review Studio 2.0 gate evidence is published in [reports/review-studio.html](reports/review-studio.html) - context budget summaries are tracked in [reports/context_budget.md](reports/context_budget.md) - portability status is tracked in [reports/portability_score.md](reports/portability_score.md) @@ -359,7 +362,7 @@ Utility scripts that make the meta-skill operational: - `governance_check.py`: validates owner, review cadence, lifecycle stage, and maturity metadata - `render_context_reports.py`: generates root and example context-budget reports plus a shared context summary - `render_regression_history.py`: turns milestone snapshots into a readable regression history report -- `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation +- `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage - `render_skill_overview.py`: generates the white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, top-right language switch, v2 scorecard, inline SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap - `export_skill_ir.py`: exports the 2.0 platform-neutral Skill IR contract from `SKILL.md`, manifest, interface metadata, evals, resources, and reports @@ -434,7 +437,7 @@ The typical flow is: 3. choose scaffold, production, or library mode 4. generate the package 5. run the sizing and trigger checks if needed -6. export target-specific compatibility artifacts +6. export target-specific compatibility artifacts from the Skill IR contract ### 3. Export compatibility artifacts diff --git a/SKILL.md b/SKILL.md index e15f63cc..ea3762c4 100644 --- a/SKILL.md +++ b/SKILL.md @@ -32,17 +32,19 @@ Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes]( 6. Add only folders and gates that earn their keep. 7. Surface the top three next iteration directions. -Core playbooks: [Method](references/skill-engineering-method.md), [Intent Dialogue](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Prompt Engineering](references/prompt-engineering-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md). +Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Review Studio](references/review-studio-method.md). + +## Skill OS 2.0 Gates + +For production, library, governed, or team-distributed work, use Skill IR, trigger + output eval, Skill Atlas, runtime conformance, trust report, and Review Studio before release. ## First-Turn Style When the skill first activates: -- open warmly, like a thoughtful teacher or design partner -- start from the user's work and desired outcome before asking for structure +- start from the user's work and desired outcome before structure - ask only `2-3` key questions unless the user already gave enough detail -- let the user answer naturally first; offer a tiny scaffold only as a shortcut -- avoid cold field lists; turn benchmark work into one recommendation unless uncertainty or conflict needs a visible call +- avoid cold field lists; surface benchmark choices only when uncertainty or conflict needs a call Chinese conversations should sound soft and companion-like rather than procedural. @@ -50,14 +52,8 @@ Opening patterns: [Intent Dialogue](references/intent-dialogue.md). ## Output Contract -Unless the user asks otherwise, produce: - -1. a working skill directory -2. a `SKILL.md` -3. aligned `agents/interface.yaml` -4. optional `references/`, `scripts/`, `evals/`, `reports/`, and `manifest.json` when justified -5. a short summary of boundary, exclusions, gates, and next steps +Unless the user asks otherwise, produce a working skill directory with `SKILL.md`, aligned `agents/interface.yaml`, justified assets, and a short summary of boundary, exclusions, gates, and next steps. ## Reference Map -Primary references: [Method](references/skill-engineering-method.md), [Reference Scan](references/reference-scan.md), [Artifact Design](references/artifact-design-doctrine.md), [Prompt Engineering](references/prompt-engineering-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md). +Primary references: [Method](references/skill-engineering-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md). diff --git a/evals/README.md b/evals/README.md index f36f226c..56d60e79 100644 --- a/evals/README.md +++ b/evals/README.md @@ -15,8 +15,8 @@ Contents: - `improved_description.txt`: current stronger trigger description - `../reports/description_optimization*.{json,md}`: generated route-optimization reports for the root skill - `failure-cases.md`: current weak spots and regression targets -- `packaging_expectations.json`: required packaging behaviors for supported targets -- `output/`: Output Eval Lab cases and schema for with-skill vs baseline assertion grading +- `packaging_expectations.json`: required packaging behaviors for supported targets, including IR provenance and semantic parity fields +- `output/`: Output Eval Lab cases, fixtures, and schema for with-skill vs baseline assertion grading - `../reports/`: generated suite JSON plus the homepage-visible family summary panel source Use: @@ -57,4 +57,4 @@ Regression scope now includes: - judge-backed blind-holdout verification that adds a rubric-based second opinion for blind prompts - calibration summaries that surface score gaps, threshold margins, and risk bands for each acceptance gate - family-level drift history that records which blind and adversarial families stay clean over time -- output-quality scorecards that compare without-skill and with-skill artifacts through assertion grading +- output-quality scorecards that compare without-skill and with-skill artifacts through assertion grading, including near-neighbor and file-backed governed cases diff --git a/evals/output/cases.jsonl b/evals/output/cases.jsonl index 70d1b6db..bb18c3c1 100644 --- a/evals/output/cases.jsonl +++ b/evals/output/cases.jsonl @@ -1,3 +1,5 @@ {"id":"skill-package-contract","prompt":"Turn this repeated workflow into a reusable team skill package.","baseline_output":"I can write a prompt for that workflow and include a short checklist.","with_skill_output":"Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/.","assertions":[{"id":"has-entrypoint","description":"Output names the routeable Skill entrypoint.","required":["SKILL.md"],"weight":1,"failure_type":"missing_entrypoint"},{"id":"has-interface","description":"Output includes neutral interface metadata.","required":["agents/interface.yaml"],"weight":1,"failure_type":"missing_interface"},{"id":"has-report","description":"Output points reviewer to generated overview report.","required":["reports/skill-overview.html"],"weight":1,"failure_type":"missing_review_artifact"},{"id":"has-resource-boundary","description":"Output preserves entrypoint/resource separation.","required":["references/","scripts/","reports/"],"weight":1,"failure_type":"resource_boundary"}],"human_review":{"expected_winner":"with_skill"}} {"id":"output-eval-expectation","prompt":"Upgrade this production skill so we know whether the generated output is better than baseline.","baseline_output":"Add more examples and run the trigger eval again.","with_skill_output":"Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md.","assertions":[{"id":"has-baseline-comparison","description":"Output explicitly compares with-skill and baseline outputs.","required":["with-skill","baseline"],"weight":1,"failure_type":"missing_baseline_delta"},{"id":"has-assertion-grading","description":"Output includes objective assertion grading.","required":["assertion grading"],"weight":1,"failure_type":"missing_assertions"},{"id":"has-scorecard","description":"Output produces a scorecard report path.","required":["reports/output_quality_scorecard.md"],"weight":1,"failure_type":"missing_scorecard"},{"id":"has-failure-taxonomy","description":"Output records failure taxonomy.","required":["failure taxonomy"],"weight":1,"failure_type":"missing_failure_taxonomy"}],"human_review":{"expected_winner":"with_skill"}} {"id":"ir-before-packaging","prompt":"Package this skill for OpenAI, Claude, Agent Skills, and generic targets.","baseline_output":"Export folders for each platform and adjust files as needed.","with_skill_output":"Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract.","assertions":[{"id":"has-ir-first","description":"Output requires Skill IR before packaging.","required":["Skill IR first"],"weight":1,"failure_type":"missing_ir"},{"id":"has-core-ir-fields","description":"Output lists core IR contract fields.","required":["job_to_be_done","trigger_surface","eval_plan","governance"],"weight":1,"failure_type":"incomplete_ir_contract"},{"id":"has-targets","description":"Output names requested runtime targets.","required":["OpenAI","Claude","Agent Skills","generic"],"weight":1,"failure_type":"missing_targets"},{"id":"has-semantic-preservation","description":"Output says targets must preserve the capability contract.","required":["preserve the same capability contract"],"weight":1,"failure_type":"semantic_drift"}],"human_review":{"expected_winner":"with_skill"}} +{"id":"near-neighbor-boundary","prompt":"I only need a one-off summary of these notes, not a reusable process.","baseline_output":"Create a SKILL.md and a reusable workflow anyway so future notes can use it.","with_skill_output":"Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract.","assertions":[{"id":"declines-skill","description":"Output refuses unnecessary skill creation for one-off work.","required":["Do not create a skill"],"weight":1,"failure_type":"over_trigger"},{"id":"names-near-neighbor","description":"Output labels the request as near-neighbor instead of owned work.","required":["near-neighbor"],"weight":1,"failure_type":"missing_boundary_language"},{"id":"requires-reuse-signal","description":"Output asks for repeat-use evidence before packaging.","required":["repeated use","reusable output contract"],"weight":1,"failure_type":"missing_reuse_gate"}],"human_review":{"expected_winner":"with_skill"},"metadata":{"case_type":"near_neighbor","tier":"production"}} +{"id":"file-backed-governed-package","prompt":"Turn the attached release brief source into a governed skill package.","input_files":["fixtures/release-brief-source.md"],"baseline_output":"Draft a release announcement with the changelog, support notes, and owner name.","with_skill_output":"Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them.","assertions":[{"id":"uses-file-backed-evidence","description":"Output names file-backed source evidence.","required":["file-backed fixture","input_files"],"weight":1,"failure_type":"missing_file_fixture"},{"id":"has-governance","description":"Output preserves governed ownership metadata.","required":["owner","review cadence"],"weight":1,"failure_type":"missing_governance"},{"id":"has-output-contract","description":"Output preserves output and rollback boundaries.","required":["output contract","rollback boundary"],"weight":1,"failure_type":"missing_contract_boundary"},{"id":"has-trust-and-scorecard","description":"Output requires trust and output scorecard artifacts.","required":["trust report","reports/output_quality_scorecard.md"],"weight":1,"failure_type":"missing_release_evidence"},{"id":"does-not-invent-evidence","description":"Output forbids invented launch evidence.","required":["missing evidence"],"forbidden":["invented launch metrics"],"weight":1,"failure_type":"invented_evidence"}],"human_review":{"expected_winner":"with_skill"},"metadata":{"case_type":"boundary","tier":"governed"}} diff --git a/evals/output/fixtures/release-brief-source.md b/evals/output/fixtures/release-brief-source.md new file mode 100644 index 00000000..8751bcf1 --- /dev/null +++ b/evals/output/fixtures/release-brief-source.md @@ -0,0 +1,14 @@ +# Release Brief Source Fixture + +## Source + +- Raw changelog notes +- Support escalation summary +- Rollback checklist +- Internal launch owner + +## Required Output Constraints + +- Produce a reusable skill package, not a one-off announcement. +- Preserve owner, review cadence, input files, output contract, and rollback boundary. +- Surface missing evidence instead of inventing launch metrics. diff --git a/evals/packaging_expectations.json b/evals/packaging_expectations.json index a03a6df7..f7766b0e 100644 --- a/evals/packaging_expectations.json +++ b/evals/packaging_expectations.json @@ -7,6 +7,11 @@ "display_name", "short_description", "default_prompt", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity", "canonical_metadata", "canonical_format", "activation_mode", diff --git a/references/packaging-contracts.md b/references/packaging-contracts.md index 8a9f6b50..a760f782 100644 --- a/references/packaging-contracts.md +++ b/references/packaging-contracts.md @@ -29,12 +29,20 @@ When `--expectations` is provided: ## Source Of Truth -The neutral source remains: +The platform-neutral semantic source is Skill IR when it exists: + +- `reports/skill-ir.json` +- `skill-ir/examples/.json` + +The structural validation sources remain: - `SKILL.md` - `agents/interface.yaml` -Target-specific metadata is generated at packaging time. +Target-specific metadata is generated at packaging time. The adapter must carry +`ir_source`, `ir_schema_version`, `job_to_be_done`, `semantic_contract`, and +`semantic_parity` so reviewers can see whether the target preserved the core +skill meaning or fell back to frontmatter-only metadata. ## Portability Model @@ -44,5 +52,6 @@ The packaging layer now preserves four portable semantics from the neutral sourc - execution - trust - degradation +- platform-neutral skill meaning from Skill IR This means portability is not just "can it export a file?" but also "does the exported target preserve the source package's activation and safety assumptions?" diff --git a/references/review-studio-method.md b/references/review-studio-method.md new file mode 100644 index 00000000..11b76144 --- /dev/null +++ b/references/review-studio-method.md @@ -0,0 +1,29 @@ +# Review Studio 2.0 Method + +Review Studio is the release-facing audit surface for a skill package. It does not replace the detailed reports; it turns them into one reviewer decision page. + +## Purpose + +- Show release blockers and warnings before the package deepens. +- Link every gate back to a concrete evidence artifact. +- Keep review flow vertical: summary first, gates second, supporting details after. +- Avoid hiding output quality, runtime, trust, and portfolio issues across separate pages. + +## Required Gates + +1. Intent Canvas: intent confidence and unresolved input/output/exclusion gaps. +2. Trigger Lab: route scorecard, misroutes, ambiguous cases, and near-neighbor safety. +3. Output Lab: with-skill vs baseline delta, case count, file-backed cases, near-neighbor cases, and boundary cases. +4. Context Budget: initial load, budget tier, warnings, and quality density. +5. Runtime Matrix: target conformance pass/fail and degradation notes. +6. Trust Report: secret scan, script surface, dependency pinning, network/interactive flags, and package hash. +7. Skill Atlas: route collisions, stale skills, owner gaps, and no-route opportunities. +8. Release Notes: promotion status, migration notes, known gaps, and next move. + +## Gate Semantics + +- `pass`: evidence is present and the gate is satisfied. +- `warn`: review can continue, but the issue must be visible before release. +- `block`: do not claim production, library, governed, or public readiness until fixed. + +For library and governed skills, Output Lab should have at least five cases and cover file-backed, near-neighbor, and boundary scenarios. diff --git a/reports/artifact-design-profile.json b/reports/artifact-design-profile.json index 7bada95b..7f520953 100644 --- a/reports/artifact-design-profile.json +++ b/reports/artifact-design-profile.json @@ -1,6 +1,6 @@ { "skill_name": "yao-meta-skill", - "design_system": "evidence frame", + "design_system": "review studio", "primary_artifact": { "key": "code_or_cli", "label": "Code, CLI, or implementation guide", @@ -33,6 +33,15 @@ ], "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." }, + { + "key": "review_viewer", + "label": "Review viewer", + "score": 1, + "matched_keywords": [ + "review" + ], + "direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix." + }, { "key": "visual_capture", "label": "Screenshot or visual evidence", @@ -82,8 +91,8 @@ "Keep the first screen useful without requiring the reader to parse every detail.", "Use tables only for comparisons; move explanations below the table.", "Keep source notes readable without flooding the body with markers.", - "Never invent missing screenshots or visual states.", - "Record source, viewport, and crop intent." + "Make differences visible instead of hiding them in prose.", + "Separate author-facing recommendations from reviewer-only evidence." ], "anti_patterns": [ "Do not copy Kami's fixed parchment background as a default.", diff --git a/reports/artifact-design-profile.md b/reports/artifact-design-profile.md index ed5736ee..000e94b8 100644 --- a/reports/artifact-design-profile.md +++ b/reports/artifact-design-profile.md @@ -1,7 +1,7 @@ # Artifact Design Profile Skill: `yao-meta-skill` -Design system: `evidence frame` +Design system: `review studio` ## Primary Artifact Direction @@ -21,6 +21,11 @@ Execution-focused technical artifact with environment assumptions, copyable comm - Score: `2` - Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail. +### Review viewer +- Matched keywords: review +- Score: `1` +- Direction: Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix. + ### Screenshot or visual evidence - Matched keywords: screenshot - Score: `1` @@ -67,8 +72,8 @@ Execution-focused technical artifact with environment assumptions, copyable comm - Keep the first screen useful without requiring the reader to parse every detail. - Use tables only for comparisons; move explanations below the table. - Keep source notes readable without flooding the body with markers. -- Never invent missing screenshots or visual states. -- Record source, viewport, and crop intent. +- Make differences visible instead of hiding them in prose. +- Separate author-facing recommendations from reviewer-only evidence. ## Anti-Patterns diff --git a/reports/context_budget.json b/reports/context_budget.json index d5794bd7..876de6ee 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -5,13 +5,13 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 810, - "other_text_tokens": 548453, - "estimated_initial_load_tokens": 986, - "estimated_total_text_tokens": 549263, - "relevant_file_count": 251, + "skill_body_tokens": 734, + "other_text_tokens": 606468, + "estimated_initial_load_tokens": 910, + "estimated_total_text_tokens": 607202, + "relevant_file_count": 241, "unused_resource_dirs": [], "quality_signal_points": 130, - "quality_density": 131.8 + "quality_density": 142.9 } } diff --git a/reports/context_budget.md b/reports/context_budget.md index 5b08c50c..1da7709c 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | -| root | `.` | `production` | 1000 | 986 | 810 | 131.8 | - | ok | +| root | `.` | `production` | 1000 | 910 | 734 | 142.9 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 5cf4efb6..efda026f 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -6,9 +6,9 @@ "path": ".", "budget_tier": "production", "budget_limit": 1000, - "initial_tokens": 986, - "skill_body_tokens": 810, - "quality_density": 131.8, + "initial_tokens": 910, + "skill_body_tokens": 734, + "quality_density": 142.9, "unused_resource_dirs": [], "ok": true, "warnings": [] diff --git a/reports/output_quality_scorecard.json b/reports/output_quality_scorecard.json index 94191d2d..bc47077f 100644 --- a/reports/output_quality_scorecard.json +++ b/reports/output_quality_scorecard.json @@ -2,7 +2,10 @@ "ok": true, "cases": "evals/output/cases.jsonl", "summary": { - "case_count": 3, + "case_count": 5, + "file_backed_case_count": 1, + "near_neighbor_case_count": 1, + "boundary_case_count": 1, "baseline_pass_rate": 0.0, "with_skill_pass_rate": 100.0, "delta": 100.0, @@ -14,6 +17,8 @@ { "id": "skill-package-contract", "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "metadata": {}, "baseline": { "score": 0.0, "passed_count": 0, @@ -166,6 +171,8 @@ { "id": "output-eval-expectation", "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "metadata": {}, "baseline": { "score": 0.0, "passed_count": 0, @@ -316,6 +323,8 @@ { "id": "ir-before-packaging", "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "metadata": {}, "baseline": { "score": 0.0, "passed_count": 0, @@ -472,6 +481,322 @@ "delta": 100.0, "winner": "with_skill", "failure_taxonomy": [] + }, + { + "id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "metadata": { + "case_type": "near_neighbor", + "tier": "production" + }, + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 3, + "checks": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0, + "failure_type": "over_trigger", + "passed": false, + "missing": [ + "Do not create a skill" + ], + "present_forbidden": [] + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0, + "failure_type": "missing_boundary_language", + "passed": false, + "missing": [ + "near-neighbor" + ], + "present_forbidden": [] + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0, + "failure_type": "missing_reuse_gate", + "passed": false, + "missing": [ + "repeated use", + "reusable output contract" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0, + "failure_type": "over_trigger", + "passed": false, + "missing": [ + "Do not create a skill" + ], + "present_forbidden": [] + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0, + "failure_type": "missing_boundary_language", + "passed": false, + "missing": [ + "near-neighbor" + ], + "present_forbidden": [] + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0, + "failure_type": "missing_reuse_gate", + "passed": false, + "missing": [ + "repeated use", + "reusable output contract" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 3, + "failed_count": 0, + "checks": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0, + "failure_type": "over_trigger", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0, + "failure_type": "missing_boundary_language", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0, + "failure_type": "missing_reuse_gate", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + }, + { + "id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "metadata": { + "case_type": "boundary", + "tier": "governed" + }, + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 5, + "checks": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0, + "failure_type": "missing_file_fixture", + "passed": false, + "missing": [ + "file-backed fixture", + "input_files" + ], + "present_forbidden": [] + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0, + "failure_type": "missing_governance", + "passed": false, + "missing": [ + "review cadence" + ], + "present_forbidden": [] + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0, + "failure_type": "missing_contract_boundary", + "passed": false, + "missing": [ + "output contract", + "rollback boundary" + ], + "present_forbidden": [] + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0, + "failure_type": "missing_release_evidence", + "passed": false, + "missing": [ + "trust report", + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0, + "failure_type": "invented_evidence", + "passed": false, + "missing": [ + "missing evidence" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0, + "failure_type": "missing_file_fixture", + "passed": false, + "missing": [ + "file-backed fixture", + "input_files" + ], + "present_forbidden": [] + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0, + "failure_type": "missing_governance", + "passed": false, + "missing": [ + "review cadence" + ], + "present_forbidden": [] + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0, + "failure_type": "missing_contract_boundary", + "passed": false, + "missing": [ + "output contract", + "rollback boundary" + ], + "present_forbidden": [] + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0, + "failure_type": "missing_release_evidence", + "passed": false, + "missing": [ + "trust report", + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0, + "failure_type": "invented_evidence", + "passed": false, + "missing": [ + "missing evidence" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 5, + "failed_count": 0, + "checks": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0, + "failure_type": "missing_file_fixture", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0, + "failure_type": "missing_governance", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0, + "failure_type": "missing_contract_boundary", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0, + "failure_type": "missing_release_evidence", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0, + "failure_type": "invented_evidence", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] } ], "failures": [], diff --git a/reports/output_quality_scorecard.md b/reports/output_quality_scorecard.md index 31ed67f2..aa1cc122 100644 --- a/reports/output_quality_scorecard.md +++ b/reports/output_quality_scorecard.md @@ -2,7 +2,7 @@ This v0 scorecard compares static without-skill and with-skill outputs using assertion grading. -- Cases: `3` +- Cases: `5` - Baseline pass rate: `0.0` - With-skill pass rate: `100.0` - Delta: `100.0` @@ -16,6 +16,8 @@ This v0 scorecard compares static without-skill and with-skill outputs using ass | skill-package-contract | 0.0 | 100.0 | 100.0 | with_skill | None | | output-eval-expectation | 0.0 | 100.0 | 100.0 | with_skill | None | | ir-before-packaging | 0.0 | 100.0 | 100.0 | with_skill | None | +| near-neighbor-boundary | 0.0 | 100.0 | 100.0 | with_skill | None | +| file-backed-governed-package | 0.0 | 100.0 | 100.0 | with_skill | None | ## Failure Taxonomy diff --git a/reports/portability_score.json b/reports/portability_score.json index b79bca6a..d8b36971 100644 --- a/reports/portability_score.json +++ b/reports/portability_score.json @@ -10,10 +10,12 @@ "trust_level": "local", "remote_inline_execution": "forbid", "degradation_coverage": 3, - "snapshot_count": 3 + "snapshot_count": 3, + "ir_source": "skill-ir/examples/yao-meta-skill.json", + "ir_schema_version": "2.0.0" }, "breakdown": { - "canonical_neutral_source": 15, + "platform_neutral_semantic_source": 15, "adapter_target_coverage": 15, "activation_metadata": 10, "execution_metadata": 10, diff --git a/reports/portability_score.md b/reports/portability_score.md index 4557c9fd..067e3b9b 100644 --- a/reports/portability_score.md +++ b/reports/portability_score.md @@ -8,10 +8,12 @@ - shell: `bash` - trust level: `local` - remote inline execution: `forbid` +- Skill IR source: `skill-ir/examples/yao-meta-skill.json` +- Skill IR schema: `2.0.0` | Component | Score | | --- | ---: | -| `canonical_neutral_source` | 15 | +| `platform_neutral_semantic_source` | 15 | | `adapter_target_coverage` | 15 | | `activation_metadata` | 10 | | `execution_metadata` | 10 | diff --git a/reports/prompt-quality-profile.json b/reports/prompt-quality-profile.json index c682117e..3c84c0de 100644 --- a/reports/prompt-quality-profile.json +++ b/reports/prompt-quality-profile.json @@ -33,17 +33,6 @@ "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." }, - { - "key": "teaching_guidance", - "label": "Teaching guidance", - "score": 1, - "matched_keywords": [ - "teach" - ], - "role_guidance": "Use a teacher role that adapts to learner level and avoids overloading the first pass.", - "task_guidance": "Explain through progressive steps, examples, and visible success checks.", - "format_guidance": "Return learner-facing sections, worked examples, checkpoints, and common mistakes." - }, { "key": "prompt_engineering", "label": "Prompt engineering", @@ -58,7 +47,7 @@ ], "complexity": { "band": "complex", - "score": 6, + "score": 5, "reason": "multiple inputs, constraints, or task families require tradeoff handling" }, "need_model": { diff --git a/reports/prompt-quality-profile.md b/reports/prompt-quality-profile.md index cb601e0d..1e9579b5 100644 --- a/reports/prompt-quality-profile.md +++ b/reports/prompt-quality-profile.md @@ -12,7 +12,7 @@ Overall quality score: `82.0/100` ## Complexity - Band: `complex` -- Score: `6` +- Score: `5` - Reason: multiple inputs, constraints, or task families require tradeoff handling ## Need Model @@ -67,13 +67,6 @@ Overall quality score: `82.0/100` - Task: Convert the job into ordered steps with validation checks and stop conditions. - Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant. -### Teaching guidance -- Score: `1` -- Keywords: teach -- Role: Use a teacher role that adapts to learner level and avoids overloading the first pass. -- Task: Explain through progressive steps, examples, and visible success checks. -- Format: Return learner-facing sections, worked examples, checkpoints, and common mistakes. - ### Prompt engineering - Score: `1` - Keywords: prompt diff --git a/reports/review-studio.html b/reports/review-studio.html new file mode 100644 index 00000000..20c8194a --- /dev/null +++ b/reports/review-studio.html @@ -0,0 +1,198 @@ + + + + + + Yao Meta Skill Review Studio 2.0 + + + + +
+
+
Review Studio 2.0
+

Yao Meta Skill

+

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+
+ 审查结论 + review + Score 87/100 +
+
+ +
+

核心指标

+
Skill IR2.0.0

4 targets in platform-neutral contract

Output Delta100.0

5 cases; 1 file-backed

Runtime5/5

target conformance pass rate

Trust0

50 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

+
+ +
+

审查闸门

+
关注

意图画布

intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
通过

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 910/1000; quality density 142.9

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
关注

信任报告

0 secrets; 50 scripts; 2 network-capable scripts

reports/security_trust_report.json 证据
关注

组合治理

12 skills; 5 route collisions; 9 owner gaps; 10 stale

reports/skill_atlas.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked

reports/promotion_decisions.json + docs/migration-v2.md 证据
+
+ +
+

阻断事项

无。

+

关注事项

  • 意图画布intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.
  • 信任报告0 secrets; 50 scripts; 2 network-capable scripts
  • 组合治理12 skills; 5 route collisions; 9 owner gaps; 10 stale
+
+ +
+
+

意图画布

+

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+
+
+

证据路径

+
  • skill_overviewreports/skill-overview.html
  • review_viewerreports/review-viewer.html
  • output_evalreports/output_quality_scorecard.md
  • runtime_conformancereports/conformance_matrix.md
  • trust_reportreports/security_trust_report.md
  • skill_atlasreports/skill_atlas.html
  • migrationdocs/migration-v2.md
+
+
+ +
+

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

+

组合治理

{'skill_count': 12, 'route_collision_count': 5, 'owner_gap_count': 9, 'stale_count': 10, 'shared_resource_count': 0, 'no_route_opportunity_count': 0}

+
+ +
+

输出实验

{'case_count': 5, 'file_backed_case_count': 1, 'near_neighbor_case_count': 1, 'boundary_case_count': 1, 'baseline_pass_rate': 0.0, 'with_skill_pass_rate': 100.0, 'delta': 100.0, 'regression_count': 0, 'gate_pass': True, 'failure_taxonomy': []}

+

发布标准

Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor 和 boundary case。

+
+ +
+

运行矩阵

{'target_count': 5, 'pass_count': 5, 'fail_count': 0}

+

上下文

initial load 910/1000; quality density 142.9

+
+ +
+

信任报告

{'scanned_files': 127, 'script_count': 50, 'secret_findings': 0, 'dependency_files': ['requirements-ci.txt'], 'network_script_count': 2, 'interactive_script_count': 0, 'package_sha256': '18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad'}

+

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

+
+ +
+

组合治理

12 skills; 5 route collisions; 9 owner gaps; 10 stale

+

下一动作

优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再做 registry 或 telemetry。

+
+ +
+

发布路线

0 promote; 3 keep current; 0 blocked

+

世界级缺口

下一阶段应继续推进 IR-first compiler、registry audit、telemetry drift loop，以及更严格的 governed trust gates。

+
+
+ + diff --git a/reports/review-studio.json b/reports/review-studio.json new file mode 100644 index 00000000..a7ee1256 --- /dev/null +++ b/reports/review-studio.json @@ -0,0 +1,4780 @@ +{ + "schema_version": "2.0", + "ok": true, + "skill_dir": ".", + "summary": { + "decision": "review", + "world_class_score": 87, + "gate_count": 8, + "blocker_count": 0, + "warning_count": 3 + }, + "gates": [ + { + "key": "intent-canvas", + "label": "意图画布", + "status": "warn", + "detail": "intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.", + "evidence": "reports/intent-confidence.json", + "link": "intent-confidence.md" + }, + { + "key": "trigger-lab", + "label": "触发实验", + "status": "pass", + "detail": "13 trigger cases; 0 misroutes; 0 ambiguous", + "evidence": "reports/route_scorecard.json", + "link": "route_scorecard.md" + }, + { + "key": "output-lab", + "label": "输出实验", + "status": "pass", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1", + "evidence": "reports/output_quality_scorecard.json", + "link": "output_quality_scorecard.md" + }, + { + "key": "context-budget", + "label": "上下文", + "status": "pass", + "detail": "initial load 910/1000; quality density 142.9", + "evidence": "reports/context_budget.json", + "link": "context_budget.md" + }, + { + "key": "runtime-matrix", + "label": "运行矩阵", + "status": "pass", + "detail": "5 / 5 targets pass", + "evidence": "reports/conformance_matrix.json", + "link": "conformance_matrix.md" + }, + { + "key": "trust-report", + "label": "信任报告", + "status": "warn", + "detail": "0 secrets; 50 scripts; 2 network-capable scripts", + "evidence": "reports/security_trust_report.json", + "link": "security_trust_report.md" + }, + { + "key": "skill-atlas", + "label": "组合治理", + "status": "warn", + "detail": "12 skills; 5 route collisions; 9 owner gaps; 10 stale", + "evidence": "reports/skill_atlas.json", + "link": "skill_atlas.html" + }, + { + "key": "release-notes", + "label": "发布路线", + "status": "pass", + "detail": "0 promote; 3 keep current; 0 blocked", + "evidence": "reports/promotion_decisions.json + docs/migration-v2.md", + "link": "promotion_decisions.md" + } + ], + "blockers": [], + "warnings": [ + { + "key": "intent-canvas", + "label": "意图画布", + "status": "warn", + "detail": "intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.", + "evidence": "reports/intent-confidence.json", + "link": "intent-confidence.md" + }, + { + "key": "trust-report", + "label": "信任报告", + "status": "warn", + "detail": "0 secrets; 50 scripts; 2 network-capable scripts", + "evidence": "reports/security_trust_report.json", + "link": "security_trust_report.md" + }, + { + "key": "skill-atlas", + "label": "组合治理", + "status": "warn", + "detail": "12 skills; 5 route collisions; 9 owner gaps; 10 stale", + "evidence": "reports/skill_atlas.json", + "link": "skill_atlas.html" + } + ], + "evidence_paths": { + "skill_overview": "reports/skill-overview.html", + "review_viewer": "reports/review-viewer.html", + "output_eval": "reports/output_quality_scorecard.md", + "runtime_conformance": "reports/conformance_matrix.md", + "trust_report": "reports/security_trust_report.md", + "skill_atlas": "reports/skill_atlas.html", + "migration": "docs/migration-v2.md" + }, + "data": { + "overview": { + "skill_summary": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "display_name": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "maturity": "governed", + "updated_at": "2026-03-31", + "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", + "audience": "Skill 作者、复用团队和后续 reviewer。", + "deliverables": [ + "SKILL.md", + "agents/interface.yaml", + "reports/skill-ir.json", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/skill_atlas.html", + "reports/review-studio.html", + "reports/skill-overview.html" + ], + "flow": [ + "输入材料", + "Skill 包体", + "可复用能力" + ] + }, + "scorecard": { + "completeness_score": { + "label": "完整度", + "score": 100, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 100, + "reasons": [ + "已生成 13 / 13 类报告证据。", + "skill-ir.json 已存在。", + "intent-dialogue.json 已存在。", + "intent-confidence.json 已存在。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 100, + "reasons": [ + "SKILL.md 约 346 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 已承载确定性逻辑。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 42, + "reasons": [ + "入口约 346 个词/字，references 约 11313 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本偏高，建议压缩入口或拆分 references。" + ] + } + }, + "capability_profile": { + "archetype": "governed", + "task_family": "Dialogue interaction", + "maturity": "governed", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "principle_model": { + "nodes": [ + { + "title": "意图澄清", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "contract_boundary": { + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "quality_review": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", + "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", + "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" + ], + "gaps": [ + "上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "review studio", + "highlights": [ + "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding." + ] + }, + "prompt_quality": { + "overall_quality_score": 82.0, + "highlights": [ + "Primary prompt task family: Dialogue interaction.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 80/100." + ] + }, + "system_model": { + "stability": { + "score": 86, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (86/100).", + "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + } + }, + "risk_governance": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 1, + "signal": "已生成 13 / 13 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 1, + "signal": "已生成 13 / 13 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 1, + "signal": "SKILL.md 约 346 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "package_assets": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 29 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 50 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 28 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 144 + } + ], + "file_count": 255, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 29 + }, + { + "label": "scripts", + "value": 50 + }, + { + "label": "evals", + "value": 28 + }, + { + "label": "reports", + "value": 144 + } + ] + }, + "iteration_roadmap": { + "items": [ + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + }, + { + "title": "Create an iteration evidence loop", + "why": "The package should show what changed and why after the first draft.", + "actions": [ + "Generate the HTML skill report and keep it aligned with the package.", + "Record reference scan choices and non-goals.", + "Capture the next iteration choice explicitly before adding more files." + ], + "unlocks": "A clearer path for the next author or reviewer." + } + ] + }, + "report_contract": { + "schema_version": "2.0", + "html_report": "reports/skill-overview.html", + "language": "zh-CN", + "default_language": "zh-CN", + "languages": [ + "zh-CN", + "en" + ], + "layout": "kami-white-audit-v2", + "nav_labels": [ + "技能概述", + "总览指标", + "能力画像", + "原理结构", + "契约边界", + "质量评估", + "风险治理", + "包体资产", + "迭代路线" + ], + "nav_labels_en": [ + "Overview", + "Metrics", + "Profile", + "Principle", + "Contract", + "Quality", + "Risk", + "Assets", + "Roadmap" + ] + }, + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "display_name": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "logic_steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + ], + "usage_steps": [ + "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + ], + "package_map": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 29 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 50 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 28 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 144 + } + ], + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", + "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", + "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" + ], + "scenarios": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." + ], + "trigger_contract": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "io_contract": { + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ] + }, + "principles": [ + { + "title": "意图澄清", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "roadmap": [ + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + }, + { + "title": "Create an iteration evidence loop", + "why": "The package should show what changed and why after the first draft.", + "actions": [ + "Generate the HTML skill report and keep it aligned with the package.", + "Record reference scan choices and non-goals.", + "Capture the next iteration choice explicitly before adding more files." + ], + "unlocks": "A clearer path for the next author or reviewer." + } + ], + "cards": [], + "introduction": [ + "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。", + "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。", + "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" + ], + "benchmark_highlights": [], + "skill_ir": { + "schema_version": "2.0.0", + "target_count": 4, + "trigger_samples": 8, + "output_eval_cases": 3 + }, + "output_quality": { + "case_count": 5, + "file_backed_case_count": 1, + "near_neighbor_case_count": 1, + "boundary_case_count": 1, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "gate_pass": true, + "failure_taxonomy": [] + }, + "runtime_conformance": { + "target_count": 5, + "pass_count": 5, + "fail_count": 0 + }, + "trust_security": { + "scanned_files": 127, + "script_count": 50, + "secret_findings": 0, + "dependency_files": [ + "requirements-ci.txt" + ], + "network_script_count": 2, + "interactive_script_count": 0, + "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" + }, + "skill_atlas": { + "skill_count": 12, + "route_collision_count": 5, + "owner_gap_count": 9, + "stale_count": 10, + "shared_resource_count": 0, + "no_route_opportunity_count": 0 + }, + "synthesis_highlights": [ + "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "artifact_design": { + "design_system": "review studio", + "highlights": [ + "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding." + ] + }, + "prompt_quality": { + "overall_quality_score": 82.0, + "highlights": [ + "Primary prompt task family: Dialogue interaction.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 80/100." + ] + }, + "system_model": { + "stability": { + "score": 86, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (86/100).", + "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + }, + "metadata": { + "canonical_format": "agent-skills", + "targets": [ + "openai", + "claude", + "generic" + ], + "maturity_tier": "governed", + "skill_archetype": "governed", + "updated_at": "2026-03-31" + } + }, + "intent_confidence": { + "score": 30, + "band": "low", + "gate_passed": false, + "strengths": [ + "The recurring job is concrete enough to anchor the package." + ], + "gaps": [ + { + "key": "real_inputs", + "label": "Real inputs are missing", + "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", + "severity": "high" + }, + { + "key": "primary_output", + "label": "Primary output is missing", + "reason": "The package does not yet know what it must hand back.", + "severity": "high" + }, + { + "key": "exclusions", + "label": "Near-neighbor exclusions are missing", + "reason": "The route may blur into nearby requests without an exclusion list.", + "severity": "high" + }, + { + "key": "constraints", + "label": "Constraints are missing", + "reason": "The package does not yet know which tradeoffs matter most.", + "severity": "high" + }, + { + "key": "standards", + "label": "Quality bar is implied, not explicit", + "reason": "The first evaluation target is still underspecified.", + "severity": "medium" + } + ], + "follow_up_questions": [ + { + "slot": "real_inputs", + "question": "What material will people actually hand to this skill in practice?", + "why": "Real input shape decides whether references, scripts, or examples are needed.", + "list": true, + "label": "Real inputs are missing", + "severity": "high" + }, + { + "slot": "primary_output", + "question": "What finished hand-back should this skill return so the next person can keep moving?", + "why": "The output is the anchor for package design and review.", + "list": false, + "label": "Primary output is missing", + "severity": "high" + }, + { + "slot": "exclusions", + "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?", + "why": "Exclusions are the fastest route to better trigger quality.", + "list": true, + "label": "Near-neighbor exclusions are missing", + "severity": "high" + } + ], + "anchor_sentence": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "context": { + "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "real_inputs": [], + "primary_output": "", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] + } + }, + "intent_dialogue": { + "skill_name": "yao-meta-skill", + "title": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "focus": "portability-and-contract", + "opening_frame": "Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that.", + "reference_note": "If you already have examples you admire, bring them in. We will learn the pattern, not copy the source.", + "conversation_path": [ + "Start with the user's own words, not package vocabulary.", + "Reflect the job, output, and non-goals back in one clean sentence.", + "Only then offer a tiny scaffold if it would help the user move faster." + ], + "opening_styles": [ + { + "label": "温柔陪伴型", + "best_when": "用户想法还散、还在试探，或者需要先被接住。", + "message": "我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。" + }, + { + "label": "专业教练型", + "best_when": "用户目标比较明确，希望被高效带着走。", + "message": "我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。" + }, + { + "label": "共创伙伴型", + "best_when": "用户已经有一些想法，希望一起打磨，不想被填表。", + "message": "我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。" + } + ], + "optional_scaffold": [ + "The repeated job it should reliably handle", + "The real inputs people will hand to it", + "The useful output it should hand back", + "What it should clearly refuse" + ], + "questions": [ + { + "question": "If this skill worked beautifully, what recurring job would it reliably handle for the user every time?", + "why": "This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape." + }, + { + "question": "When someone reaches for this skill in the real world, what materials will they actually hand to it?", + "why": "Input shape decides whether references, scripts, or templates are needed." + }, + { + "question": "What finished output should it hand back so the user can immediately keep moving?", + "why": "Outputs should drive the package structure before extra guidance is added." + }, + { + "question": "Which nearby requests should this skill politely refuse so the boundary stays clean?", + "why": "The exclusion list is the fastest route to better trigger quality." + }, + { + "question": "What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?", + "why": "Constraints decide how much structure, packaging, and review the skill actually needs." + }, + { + "question": "Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example?", + "why": "A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material." + }, + { + "question": "Which environments or clients must be able to consume this skill?", + "why": "This sets the minimum metadata and degradation strategy." + } + ], + "output": { + "capability_sentence": "Yao Meta Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily.", + "required_capture": [ + "recurring job", + "real inputs", + "required outputs", + "exclusions", + "constraints", + "reference preferences", + "first evaluation target" + ], + "recommended_first_gate": "portability and contract" + } + }, + "route_scorecard": { + "summary": { + "total_cases": 13, + "correct_cases": 13, + "accuracy": 1.0, + "ambiguous_case_count": 0, + "margin_warning_threshold": 0.08, + "misroute_count": 0, + "no_route_accuracy": 1.0 + }, + "route_stats": { + "yao-meta-skill": { + "expected_count": 3, + "predicted_count": 3, + "correct_count": 3, + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.611 + }, + "team-frontend-review": { + "expected_count": 3, + "predicted_count": 3, + "correct_count": 3, + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.805 + }, + "governed-incident-command": { + "expected_count": 3, + "predicted_count": 3, + "correct_count": 3, + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.659 + }, + "no_route": { + "expected_count": 4, + "predicted_count": 4, + "correct_count": 4, + "precision": 1.0, + "recall": 1.0, + "average_margin": null + } + }, + "confusion_matrix": { + "yao-meta-skill": { + "yao-meta-skill": 3, + "team-frontend-review": 0, + "governed-incident-command": 0, + "no_route": 0 + }, + "team-frontend-review": { + "yao-meta-skill": 0, + "team-frontend-review": 3, + "governed-incident-command": 0, + "no_route": 0 + }, + "governed-incident-command": { + "yao-meta-skill": 0, + "team-frontend-review": 0, + "governed-incident-command": 3, + "no_route": 0 + }, + "no_route": { + "yao-meta-skill": 0, + "team-frontend-review": 0, + "governed-incident-command": 0, + "no_route": 4 + } + }, + "ambiguous_cases": [], + "misroutes": [], + "cases": [ + { + "family": "meta_create", + "prompt": "Create a reusable skill from this workflow and add evals before sharing it with the team.", + "expected_route": "yao-meta-skill", + "predicted_route": "yao-meta-skill", + "correct": true, + "margin": 0.741, + "ambiguous": false, + "ranked_routes": [ + { + "name": "yao-meta-skill", + "score": 0.741, + "passed_threshold": true, + "threshold": 0.33, + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.01, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.01, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "meta_iterate", + "prompt": "Refactor this existing skill description, tighten the trigger boundary, and package it for reuse.", + "expected_route": "yao-meta-skill", + "predicted_route": "yao-meta-skill", + "correct": true, + "margin": 0.535, + "ambiguous": false, + "ranked_routes": [ + { + "name": "yao-meta-skill", + "score": 0.535, + "passed_threshold": true, + "threshold": 0.33, + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package" + ], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.011, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.01, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "meta_operationalize", + "prompt": "Turn these prompts, notes, and transcripts into a maintained skill package with references and scripts.", + "expected_route": "yao-meta-skill", + "predicted_route": "yao-meta-skill", + "correct": true, + "margin": 0.558, + "ambiguous": false, + "ranked_routes": [ + { + "name": "yao-meta-skill", + "score": 0.558, + "passed_threshold": true, + "threshold": 0.33, + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.269, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [ + "governed_workflow" + ], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.011, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "frontend_premerge", + "prompt": "Run a pre-merge frontend review for keyboard navigation issues and unsafe rendering.", + "expected_route": "team-frontend-review", + "predicted_route": "team-frontend-review", + "correct": true, + "margin": 0.893, + "ambiguous": false, + "ranked_routes": [ + { + "name": "team-frontend-review", + "score": 0.893, + "passed_threshold": true, + "threshold": 0.48, + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "review_frontend", + "ui_security", + "ux_regression" + ], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.018, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.012, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "frontend_states", + "prompt": "Run a pre-merge frontend review on this React diff for missing empty states, form handling, and UX regressions.", + "expected_route": "team-frontend-review", + "predicted_route": "team-frontend-review", + "correct": true, + "margin": 0.704, + "ambiguous": false, + "ranked_routes": [ + { + "name": "team-frontend-review", + "score": 0.704, + "passed_threshold": true, + "threshold": 0.48, + "matched_desired_concepts": [ + "frontend_scope", + "review_frontend", + "ui_security", + "ux_regression" + ], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.013, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.008, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "frontend_a11y", + "prompt": "Review React components for a11y regressions, unsafe rendering, and UI behavior before merge.", + "expected_route": "team-frontend-review", + "predicted_route": "team-frontend-review", + "correct": true, + "margin": 0.819, + "ambiguous": false, + "ranked_routes": [ + { + "name": "team-frontend-review", + "score": 0.819, + "passed_threshold": true, + "threshold": 0.48, + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "review_frontend", + "ui_security" + ], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.012, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.006, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "incident_packet", + "prompt": "Build a governed incident command packet from alerts, responder notes, incident communication needs, and severity findings.", + "expected_route": "governed-incident-command", + "predicted_route": "governed-incident-command", + "correct": true, + "margin": 1.0, + "ambiguous": false, + "ranked_routes": [ + { + "name": "governed-incident-command", + "score": 1.0, + "passed_threshold": true, + "threshold": 0.48, + "matched_desired_concepts": [ + "comms_and_owners", + "governed_workflow", + "severity_assessment" + ], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.021, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.011, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "incident_severity", + "prompt": "Run severity assessment and package incident communication from this outage review.", + "expected_route": "governed-incident-command", + "predicted_route": "governed-incident-command", + "correct": true, + "margin": 0.805, + "ambiguous": false, + "ranked_routes": [ + { + "name": "governed-incident-command", + "score": 0.805, + "passed_threshold": true, + "threshold": 0.48, + "matched_desired_concepts": [ + "comms_and_owners", + "severity_assessment" + ], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.189, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [ + "reuse_package" + ], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.022, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "incident_governed_workflow", + "prompt": "Standardize this incident review into a reusable governed workflow with owner actions.", + "expected_route": "governed-incident-command", + "predicted_route": "governed-incident-command", + "correct": true, + "margin": 0.172, + "ambiguous": false, + "ranked_routes": [ + { + "name": "governed-incident-command", + "score": 0.62, + "passed_threshold": true, + "threshold": 0.48, + "matched_desired_concepts": [ + "comms_and_owners", + "governed_workflow" + ], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.448, + "passed_threshold": true, + "threshold": 0.33, + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.013, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "no_route_explain", + "prompt": "Explain keyboard accessibility basics in plain English.", + "expected_route": "no_route", + "predicted_route": "no_route", + "correct": true, + "margin": null, + "ambiguous": false, + "ranked_routes": [ + { + "name": "team-frontend-review", + "score": 0.195, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [ + "accessibility" + ], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.0, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ] + }, + { + "name": "governed-incident-command", + "score": 0.0, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "no_route_build", + "prompt": "Build a React form page with loading and empty states.", + "expected_route": "no_route", + "predicted_route": "no_route", + "correct": true, + "margin": null, + "ambiguous": false, + "ranked_routes": [ + { + "name": "yao-meta-skill", + "score": 0.016, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.008, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.0, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [ + "frontend_scope" + ], + "matched_negative_concepts": [ + "build_ui" + ] + } + ] + }, + { + "family": "no_route_single_update", + "prompt": "Write one short outage update for customers from these notes.", + "expected_route": "no_route", + "predicted_route": "no_route", + "correct": true, + "margin": null, + "ambiguous": false, + "ranked_routes": [ + { + "name": "yao-meta-skill", + "score": 0.024, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.008, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "governed-incident-command", + "score": 0.0, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + } + ] + }, + { + "family": "no_route_document_only", + "prompt": "Summarize this workflow and keep it as documentation only.", + "expected_route": "no_route", + "predicted_route": "no_route", + "correct": true, + "margin": null, + "ambiguous": false, + "ranked_routes": [ + { + "name": "governed-incident-command", + "score": 0.01, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "team-frontend-review", + "score": 0.009, + "passed_threshold": false, + "threshold": 0.48, + "matched_desired_concepts": [], + "matched_negative_concepts": [] + }, + { + "name": "yao-meta-skill", + "score": 0.0, + "passed_threshold": false, + "threshold": 0.33, + "matched_desired_concepts": [ + "transform_workflow" + ], + "matched_negative_concepts": [ + "no_build_directive", + "summarize_only" + ] + } + ] + } + ] + }, + "output_quality": { + "ok": true, + "cases": "evals/output/cases.jsonl", + "summary": { + "case_count": 5, + "file_backed_case_count": 1, + "near_neighbor_case_count": 1, + "boundary_case_count": 1, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "gate_pass": true, + "failure_taxonomy": [] + }, + "results": [ + { + "id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "input_files": [], + "metadata": {}, + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "checks": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0, + "failure_type": "missing_entrypoint", + "passed": false, + "missing": [ + "SKILL.md" + ], + "present_forbidden": [] + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0, + "failure_type": "missing_interface", + "passed": false, + "missing": [ + "agents/interface.yaml" + ], + "present_forbidden": [] + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0, + "failure_type": "missing_review_artifact", + "passed": false, + "missing": [ + "reports/skill-overview.html" + ], + "present_forbidden": [] + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0, + "failure_type": "resource_boundary", + "passed": false, + "missing": [ + "references/", + "scripts/", + "reports/" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0, + "failure_type": "missing_entrypoint", + "passed": false, + "missing": [ + "SKILL.md" + ], + "present_forbidden": [] + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0, + "failure_type": "missing_interface", + "passed": false, + "missing": [ + "agents/interface.yaml" + ], + "present_forbidden": [] + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0, + "failure_type": "missing_review_artifact", + "passed": false, + "missing": [ + "reports/skill-overview.html" + ], + "present_forbidden": [] + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0, + "failure_type": "resource_boundary", + "passed": false, + "missing": [ + "references/", + "scripts/", + "reports/" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "checks": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0, + "failure_type": "missing_entrypoint", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0, + "failure_type": "missing_interface", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0, + "failure_type": "missing_review_artifact", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0, + "failure_type": "resource_boundary", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + }, + { + "id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "input_files": [], + "metadata": {}, + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "checks": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0, + "failure_type": "missing_baseline_delta", + "passed": false, + "missing": [ + "with-skill", + "baseline" + ], + "present_forbidden": [] + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0, + "failure_type": "missing_assertions", + "passed": false, + "missing": [ + "assertion grading" + ], + "present_forbidden": [] + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0, + "failure_type": "missing_scorecard", + "passed": false, + "missing": [ + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0, + "failure_type": "missing_failure_taxonomy", + "passed": false, + "missing": [ + "failure taxonomy" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0, + "failure_type": "missing_baseline_delta", + "passed": false, + "missing": [ + "with-skill", + "baseline" + ], + "present_forbidden": [] + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0, + "failure_type": "missing_assertions", + "passed": false, + "missing": [ + "assertion grading" + ], + "present_forbidden": [] + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0, + "failure_type": "missing_scorecard", + "passed": false, + "missing": [ + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0, + "failure_type": "missing_failure_taxonomy", + "passed": false, + "missing": [ + "failure taxonomy" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "checks": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0, + "failure_type": "missing_baseline_delta", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0, + "failure_type": "missing_assertions", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0, + "failure_type": "missing_scorecard", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0, + "failure_type": "missing_failure_taxonomy", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + }, + { + "id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "input_files": [], + "metadata": {}, + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "checks": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0, + "failure_type": "missing_ir", + "passed": false, + "missing": [ + "Skill IR first" + ], + "present_forbidden": [] + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0, + "failure_type": "incomplete_ir_contract", + "passed": false, + "missing": [ + "job_to_be_done", + "trigger_surface", + "eval_plan", + "governance" + ], + "present_forbidden": [] + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0, + "failure_type": "missing_targets", + "passed": false, + "missing": [ + "OpenAI", + "Claude", + "Agent Skills", + "generic" + ], + "present_forbidden": [] + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0, + "failure_type": "semantic_drift", + "passed": false, + "missing": [ + "preserve the same capability contract" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0, + "failure_type": "missing_ir", + "passed": false, + "missing": [ + "Skill IR first" + ], + "present_forbidden": [] + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0, + "failure_type": "incomplete_ir_contract", + "passed": false, + "missing": [ + "job_to_be_done", + "trigger_surface", + "eval_plan", + "governance" + ], + "present_forbidden": [] + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0, + "failure_type": "missing_targets", + "passed": false, + "missing": [ + "OpenAI", + "Claude", + "Agent Skills", + "generic" + ], + "present_forbidden": [] + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0, + "failure_type": "semantic_drift", + "passed": false, + "missing": [ + "preserve the same capability contract" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "checks": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0, + "failure_type": "missing_ir", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0, + "failure_type": "incomplete_ir_contract", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0, + "failure_type": "missing_targets", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0, + "failure_type": "semantic_drift", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + }, + { + "id": "near-neighbor-boundary", + "prompt": "I only need a one-off summary of these notes, not a reusable process.", + "input_files": [], + "metadata": { + "case_type": "near_neighbor", + "tier": "production" + }, + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 3, + "checks": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0, + "failure_type": "over_trigger", + "passed": false, + "missing": [ + "Do not create a skill" + ], + "present_forbidden": [] + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0, + "failure_type": "missing_boundary_language", + "passed": false, + "missing": [ + "near-neighbor" + ], + "present_forbidden": [] + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0, + "failure_type": "missing_reuse_gate", + "passed": false, + "missing": [ + "repeated use", + "reusable output contract" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0, + "failure_type": "over_trigger", + "passed": false, + "missing": [ + "Do not create a skill" + ], + "present_forbidden": [] + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0, + "failure_type": "missing_boundary_language", + "passed": false, + "missing": [ + "near-neighbor" + ], + "present_forbidden": [] + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0, + "failure_type": "missing_reuse_gate", + "passed": false, + "missing": [ + "repeated use", + "reusable output contract" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 3, + "failed_count": 0, + "checks": [ + { + "id": "declines-skill", + "description": "Output refuses unnecessary skill creation for one-off work.", + "weight": 1.0, + "failure_type": "over_trigger", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "names-near-neighbor", + "description": "Output labels the request as near-neighbor instead of owned work.", + "weight": 1.0, + "failure_type": "missing_boundary_language", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "requires-reuse-signal", + "description": "Output asks for repeat-use evidence before packaging.", + "weight": 1.0, + "failure_type": "missing_reuse_gate", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + }, + { + "id": "file-backed-governed-package", + "prompt": "Turn the attached release brief source into a governed skill package.", + "input_files": [ + "fixtures/release-brief-source.md" + ], + "metadata": { + "case_type": "boundary", + "tier": "governed" + }, + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 5, + "checks": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0, + "failure_type": "missing_file_fixture", + "passed": false, + "missing": [ + "file-backed fixture", + "input_files" + ], + "present_forbidden": [] + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0, + "failure_type": "missing_governance", + "passed": false, + "missing": [ + "review cadence" + ], + "present_forbidden": [] + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0, + "failure_type": "missing_contract_boundary", + "passed": false, + "missing": [ + "output contract", + "rollback boundary" + ], + "present_forbidden": [] + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0, + "failure_type": "missing_release_evidence", + "passed": false, + "missing": [ + "trust report", + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0, + "failure_type": "invented_evidence", + "passed": false, + "missing": [ + "missing evidence" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0, + "failure_type": "missing_file_fixture", + "passed": false, + "missing": [ + "file-backed fixture", + "input_files" + ], + "present_forbidden": [] + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0, + "failure_type": "missing_governance", + "passed": false, + "missing": [ + "review cadence" + ], + "present_forbidden": [] + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0, + "failure_type": "missing_contract_boundary", + "passed": false, + "missing": [ + "output contract", + "rollback boundary" + ], + "present_forbidden": [] + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0, + "failure_type": "missing_release_evidence", + "passed": false, + "missing": [ + "trust report", + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0, + "failure_type": "invented_evidence", + "passed": false, + "missing": [ + "missing evidence" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 5, + "failed_count": 0, + "checks": [ + { + "id": "uses-file-backed-evidence", + "description": "Output names file-backed source evidence.", + "weight": 1.0, + "failure_type": "missing_file_fixture", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-governance", + "description": "Output preserves governed ownership metadata.", + "weight": 1.0, + "failure_type": "missing_governance", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-output-contract", + "description": "Output preserves output and rollback boundaries.", + "weight": 1.0, + "failure_type": "missing_contract_boundary", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-trust-and-scorecard", + "description": "Output requires trust and output scorecard artifacts.", + "weight": 1.0, + "failure_type": "missing_release_evidence", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "does-not-invent-evidence", + "description": "Output forbids invented launch evidence.", + "weight": 1.0, + "failure_type": "invented_evidence", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + } + ], + "failures": [], + "artifacts": { + "json": "reports/output_quality_scorecard.json", + "markdown": "reports/output_quality_scorecard.md" + } + }, + "conformance": { + "ok": true, + "skill": "yao-meta-skill", + "targets": [ + { + "target": "openai", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares openai", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "adapter target declares openai", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "openai degradation note exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares claude", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "adapter target declares claude", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "claude degradation note exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [] + }, + { + "target": "agent-skills", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "directory name matches skill name", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares agent-skills via agent-skills-compatible", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "canonical format is agent-skills", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [ + "agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + ] + }, + { + "target": "vscode", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "directory name matches skill name", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares vscode via agent-skills-compatible", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "canonical format is agent-skills", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [ + "vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + ] + }, + { + "target": "generic", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares generic", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "adapter target declares generic", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "generic degradation note exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [] + } + ], + "summary": { + "target_count": 5, + "pass_count": 5, + "fail_count": 0 + }, + "artifacts": { + "json": "reports/conformance_matrix.json", + "markdown": "reports/conformance_matrix.md" + } + }, + "trust": { + "ok": true, + "skill_dir": ".", + "summary": { + "scanned_files": 127, + "script_count": 50, + "secret_findings": 0, + "dependency_files": [ + "requirements-ci.txt" + ], + "network_script_count": 2, + "interactive_script_count": 0, + "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" + }, + "failures": [], + "warnings": [ + "Scripts without argparse/help surface: scripts/render_context_reports.py, scripts/render_social_preview.py, scripts/skill_report_charts.py, scripts/skill_report_metrics.py, scripts/skill_report_model.py", + "Network-capable scripts require bounded host policy: scripts/check_update.py, scripts/github_benchmark_scan.py" + ], + "secrets": [], + "scripts": [ + { + "path": "scripts/build_confusion_matrix.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/build_skill_atlas.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/check_update.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": true, + "uses_subprocess": false + }, + { + "path": "scripts/ci_test.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/collect_feedback.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/context_sizer.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/create_iteration_snapshot.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/cross_packager.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/diff_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/export_skill_ir.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/github_benchmark_scan.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": true, + "uses_subprocess": false + }, + { + "path": "scripts/governance_check.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/init_skill.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/judge_blind_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/lint_skill.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/optimize_description.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/promotion_checker.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_artifact_design_profile.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_baseline_compare.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_context_reports.py", + "has_argparse": false, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_description_drift_history.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_eval_dashboard.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/render_intent_confidence.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_intent_dialogue.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_iteration_directions.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_iteration_ledger.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_output_risk_profile.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_portability_report.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_prompt_quality_profile.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_reference_scan.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_reference_synthesis.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_regression_history.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_review_studio.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_review_viewer.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_skill_overview.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_social_preview.py", + "has_argparse": false, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_system_model.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/resource_boundary_check.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/run_conformance_suite.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/run_description_optimization_suite.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/run_eval_suite.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/run_output_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/skill_report_charts.py", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/skill_report_metrics.py", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/skill_report_model.py", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/sync_local_install.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/trigger_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/trust_check.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/validate_skill.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/yao.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + } + ], + "dependencies": { + "present": [ + "requirements-ci.txt" + ], + "pinned": [ + "PyYAML==6.0.3" + ], + "unpinned": [] + }, + "trust_metadata": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "artifacts": { + "json": "reports/security_trust_report.json", + "markdown": "reports/security_trust_report.md" + } + }, + "context_budget": { + "ok": true, + "failures": [], + "warnings": [], + "stats": { + "context_budget_tier": "production", + "context_budget_limit": 1000, + "skill_body_tokens": 734, + "other_text_tokens": 606468, + "estimated_initial_load_tokens": 910, + "estimated_total_text_tokens": 607202, + "relevant_file_count": 241, + "unused_resource_dirs": [], + "quality_signal_points": 130, + "quality_density": 142.9 + } + }, + "promotion": { + "ok": true, + "summary": { + "promote": 0, + "keep_current": 3, + "blocked": 0 + }, + "targets": [ + { + "name": "yao-meta-skill", + "winner_label": "Current", + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "winner_tokens": 65, + "current_tokens": 65, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.33, + "mean_positive_score": 0.673, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "blind_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.028, + "score_gap": 0.364, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.302, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "adversarial_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", + "ok": true, + "promotion": { + "decision": "keep_current", + "winner_changed": false, + "all_gates_pass": true, + "route_clean": true, + "judge_agreement_ok": true, + "causes": [ + "no_candidate_outperformed_current" + ], + "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", + "gate_status": { + "visible_holdout_non_regression": true, + "blind_holdout_non_regression": true, + "judge_blind_non_regression": true, + "judge_blind_agreement": true, + "adversarial_non_regression": true, + "adversarial_risk_ok": true, + "route_confusion_clean": true, + "family_stability": true + }, + "route_metrics": { + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.611, + "misroute_count": 0, + "ambiguous_case_count": 0 + } + }, + "registry": [ + { + "name": "yao-meta-skill", + "role": "baseline", + "label": "Baseline", + "ranking_state": "reference", + "promotion_state": "reference", + "description": "Create and improve agent skills.", + "tokens": 8, + "dev_errors": 1, + "holdout_errors": 0 + }, + { + "name": "yao-meta-skill", + "role": "current", + "label": "Current", + "ranking_state": "selected_by_dev", + "promotion_state": "kept_current", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "tokens": 65, + "dev_errors": 0, + "holdout_errors": 0, + "reason_tags": [] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Minimal", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "tokens": 41, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", + "tokens": 56, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Balanced", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "tokens": 60, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Artifact Aware", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "tokens": 77, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit", + "longer_without_gain" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Boundary", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", + "tokens": 83, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit", + "longer_without_gain" + ] + } + ], + "decision": "keep_current" + }, + { + "name": "team-frontend-review", + "winner_label": "Current", + "winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "winner_tokens": 50, + "current_tokens": 50, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.685, + "mean_non_trigger_score": 0.089, + "mean_near_neighbor_score": 0.045, + "min_positive_score": 0.496, + "max_non_trigger_score": 0.211, + "score_gap": 0.285, + "threshold_margin": 0.016, + "positive_threshold_buffer": 0.016, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.685, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.226, + "min_positive_score": 0.504, + "max_non_trigger_score": 0.416, + "score_gap": 0.088, + "threshold_margin": 0.024, + "positive_threshold_buffer": 0.024, + "negative_threshold_buffer": 0.064, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.71, + "mean_non_trigger_score": 0.009, + "mean_near_neighbor_score": 0.018, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.035, + "score_gap": 0.509, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.305, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", + "ok": true, + "promotion": { + "decision": "keep_current", + "winner_changed": false, + "all_gates_pass": true, + "route_clean": true, + "judge_agreement_ok": true, + "causes": [ + "no_candidate_outperformed_current", + "current_holdout_risk" + ], + "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", + "gate_status": { + "visible_holdout_non_regression": true, + "blind_holdout_non_regression": true, + "judge_blind_non_regression": true, + "judge_blind_agreement": true, + "adversarial_non_regression": true, + "adversarial_risk_ok": true, + "route_confusion_clean": true, + "family_stability": true + }, + "route_metrics": { + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.805, + "misroute_count": 0, + "ambiguous_case_count": 0 + } + }, + "registry": [ + { + "name": "team-frontend-review", + "role": "baseline", + "label": "Baseline", + "ranking_state": "reference", + "promotion_state": "reference", + "description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.", + "tokens": 52, + "dev_errors": 3, + "holdout_errors": 0 + }, + { + "name": "team-frontend-review", + "role": "current", + "label": "Current", + "ranking_state": "selected_by_dev", + "promotion_state": "kept_current", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "tokens": 50, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.", + "tokens": 62, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Balanced", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "tokens": 64, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Artifact Aware", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Trigger when requests mention React components, forms, frontend diffs, or UI code and the job is to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "tokens": 84, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Boundary", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.", + "tokens": 90, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + } + ], + "decision": "keep_current" + }, + { + "name": "governed-incident-command", + "winner_label": "Current", + "winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "winner_tokens": 37, + "current_tokens": 37, + "winner_holdout_fp": 0, + "winner_holdout_fn": 1, + "current_holdout_fp": 0, + "current_holdout_fn": 1, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 2, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 1, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.37, + "mean_non_trigger_score": 0.019, + "mean_near_neighbor_score": 0.037, + "min_positive_score": 0.08, + "max_non_trigger_score": 0.057, + "score_gap": 0.023, + "threshold_margin": -0.4, + "positive_threshold_buffer": -0.4, + "negative_threshold_buffer": 0.423, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "overlap" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.902, + "mean_non_trigger_score": 0.168, + "mean_near_neighbor_score": 0.138, + "min_positive_score": 0.804, + "max_non_trigger_score": 0.386, + "score_gap": 0.418, + "threshold_margin": 0.094, + "positive_threshold_buffer": 0.324, + "negative_threshold_buffer": 0.094, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "healthy" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.827, + "mean_non_trigger_score": 0.027, + "mean_near_neighbor_score": 0.011, + "min_positive_score": 0.654, + "max_non_trigger_score": 0.056, + "score_gap": 0.598, + "threshold_margin": 0.284, + "positive_threshold_buffer": 0.314, + "negative_threshold_buffer": 0.284, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.69, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", + "ok": true, + "promotion": { + "decision": "keep_current", + "winner_changed": false, + "all_gates_pass": true, + "route_clean": true, + "judge_agreement_ok": true, + "causes": [ + "no_candidate_outperformed_current", + "current_holdout_gap_present", + "current_holdout_risk" + ], + "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", + "gate_status": { + "visible_holdout_non_regression": true, + "blind_holdout_non_regression": true, + "judge_blind_non_regression": true, + "judge_blind_agreement": true, + "adversarial_non_regression": true, + "adversarial_risk_ok": true, + "route_confusion_clean": true, + "family_stability": true + }, + "route_metrics": { + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.659, + "misroute_count": 0, + "ambiguous_case_count": 0 + } + }, + "registry": [ + { + "name": "governed-incident-command", + "role": "baseline", + "label": "Baseline", + "ranking_state": "reference", + "promotion_state": "reference", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", + "tokens": 93, + "dev_errors": 1, + "holdout_errors": 2 + }, + { + "name": "governed-incident-command", + "role": "current", + "label": "Current", + "ranking_state": "selected_by_dev", + "promotion_state": "kept_current", + "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "tokens": 37, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, + "dev_errors": 1, + "holdout_errors": 2, + "reason_tags": [ + "visible_holdout_regression", + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Balanced", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "tokens": 54, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Boundary", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 78, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Artifact Aware", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", + "tokens": 78, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + } + ], + "decision": "keep_current" + } + ] + }, + "atlas": { + "ok": true, + "workspace_root": ".", + "summary": { + "skill_count": 12, + "route_collision_count": 5, + "owner_gap_count": 9, + "stale_count": 10, + "shared_resource_count": 0, + "no_route_opportunity_count": 0 + }, + "catalog": { + "workspace_root": ".", + "generated_at": "2026-06-13", + "skills": [ + { + "name": "yao-meta-skill", + "path": ".", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "owner": "Yao Team", + "version": "1.1.0", + "status": "active", + "maturity": "governed", + "updated_at": "2026-03-31", + "review_cadence": "quarterly", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "resources": [ + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/optimize_description.py", + "scripts/promotion_checker.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_studio.py", + "scripts/render_review_viewer.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/validate_skill.py", + "scripts/yao.py", + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/review-studio-method.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/trust-security-method.md", + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "token_set": [ + "add", + "asked", + "docs", + "evals", + "evaluate", + "existing", + "improve", + "notes", + "or", + "package", + "process", + "prompts", + "refactor", + "repeated", + "reuse", + "team", + "transcripts", + "workflows" + ] + }, + { + "name": "release-orchestrator", + "path": "examples/complex-release-orchestrator/generated-skill", + "description": "Coordinate software release preparation, rollout readiness, migration notes, rollback planning, stakeholder communication, and final go/no-go release packets. Use when asked to prepare a release packet, review release readiness, or turn scattered launch notes into one reusable release workflow. Do not use for generic project planning, one-off announcement drafting, or high-level release theory.", + "owner": "Yao Team", + "version": "1.0.0", + "status": "active", + "maturity": "production", + "updated_at": "2026-03-31", + "review_cadence": "quarterly", + "targets": [], + "resources": [ + "scripts/build_release_packet.py", + "references/migration-template.md", + "references/release-checklist.md", + "references/risk-matrix.md", + "references/stakeholder-comms.md" + ], + "token_set": [ + "announcement", + "asked", + "communication", + "coordinate", + "do", + "drafting", + "final", + "generic", + "go", + "high-level", + "launch", + "migration", + "no-go", + "not", + "notes", + "one", + "one-off", + "or", + "packet", + "packets", + "planning", + "preparation", + "prepare", + "project", + "readiness", + "release", + "review", + "rollback", + "rollout", + "scattered", + "software", + "stakeholder", + "theory", + "workflow" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "a11y", + "accessibility", + "asked", + "code", + "common", + "components", + "forms", + "frontend", + "issues", + "missing", + "or", + "pre-merge", + "quality", + "react", + "regressions", + "review", + "risky", + "security", + "states", + "ui", + "ux" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "description": "Review frontend code before merge.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "before", + "code", + "frontend", + "merge", + "review" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "description": "Review frontend code for accessibility, risky UI security issues, missing loading and error states, and pre-merge regressions. Use when asked to review React, UI, forms, frontend components, or merge-readiness for the frontend.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "accessibility", + "asked", + "code", + "components", + "error", + "forms", + "frontend", + "issues", + "loading", + "merge-readiness", + "missing", + "or", + "pre-merge", + "react", + "regressions", + "review", + "risky", + "security", + "states", + "ui" + ] + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill", + "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "owner": "Incident Operations Team", + "version": "1.0.0", + "status": "active", + "maturity": "governed", + "updated_at": "2026-03-31", + "review_cadence": "monthly", + "targets": [ + "openai", + "claude", + "generic" + ], + "resources": [ + "scripts/build_incident_packet.py", + "references/comms-policy.md", + "references/review-policy.md", + "references/severity-matrix.md" + ], + "token_set": [ + "asked", + "assemble", + "assessment", + "build", + "command", + "communication", + "governed", + "incident", + "or", + "packets", + "review", + "run", + "severity", + "standardize" + ] + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "description": "Clean messy meeting notes into structured markdown summaries. Use when asked to organize meeting notes, extract action items, separate decisions from open questions, or turn rough notes into a clean recap.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "action", + "asked", + "clean", + "decisions", + "extract", + "items", + "markdown", + "meeting", + "messy", + "notes", + "open", + "or", + "organize", + "questions", + "recap", + "rough", + "separate", + "structured", + "summaries" + ] + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [ + "references/checklist.md" + ], + "token_set": [ + "a11y", + "accessibility", + "asked", + "check", + "code", + "components", + "frontend", + "missing", + "or", + "pre-merge", + "react", + "regressions", + "rendering", + "review", + "run", + "security", + "states", + "ui", + "unsafe", + "ux" + ] + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "description": "Create GEO ranking-article packs from brand briefs. Use when asked to research a brand, find GEO or 生成式引擎优化 keywords and five competitors, generate timely titles, write five ~2000-character articles, and save them to a fixed knowledge-base folder with an HTML index. Do not use for generic SEO advice, one-off copy, or writing without competitor research.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [ + "scripts/build_html_index.py", + "references/article-blueprint.md", + "references/feishu-benchmark-patterns.md", + "references/geo-research-playbook.md", + "templates/brand-brief-template.md" + ], + "token_set": [ + "2000-character", + "advice", + "articles", + "asked", + "brand", + "briefs", + "competitor", + "competitors", + "copy", + "do", + "find", + "five", + "fixed", + "folder", + "generate", + "generic", + "geo", + "html", + "index", + "keywords", + "knowledge-base", + "not", + "one-off", + "or", + "packs", + "ranking-article", + "research", + "save", + "seo", + "them", + "timely", + "titles", + "without", + "write", + "writing", + "生成式引擎优化" + ] + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "description": "Test fixture for governance validation.", + "owner": "", + "version": "1.0.0", + "status": "frozen", + "maturity": "governed", + "updated_at": "2026/03/31", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "fixture", + "governance", + "test", + "validation" + ] + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "description": "Broken YAML fixture for packager failure testing.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "broken", + "failure", + "fixture", + "packager", + "testing", + "yaml" + ] + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "description": "Broken fixture for packager failure testing.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "broken", + "failure", + "fixture", + "packager", + "testing" + ] + } + ], + "summary": { + "skill_count": 12, + "route_collision_count": 5, + "owner_gap_count": 9, + "stale_count": 10, + "shared_resource_count": 0, + "no_route_opportunity_count": 0 + } + }, + "route_collisions": [ + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/final", + "path_b": "examples/evolution-frontend-review/v1", + "score": 0.708, + "status": "collision" + }, + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/final", + "path_b": "examples/team-frontend-review/generated-skill", + "score": 0.64, + "status": "collision" + }, + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/v1", + "path_b": "examples/team-frontend-review/generated-skill", + "score": 0.538, + "status": "collision" + }, + { + "skill_a": "broken-yaml-skill", + "skill_b": "broken-skill", + "path_a": "tests/fixtures/package_invalid_yaml", + "path_b": "tests/fixtures/package_missing_interface_field", + "score": 0.833, + "status": "collision" + }, + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/final", + "path_b": "examples/evolution-frontend-review/v0", + "score": 1.0, + "status": "duplicate-name" + } + ], + "dependency_graph": { + "nodes": [ + { + "name": "yao-meta-skill", + "path": "." + }, + { + "name": "release-orchestrator", + "path": "examples/complex-release-orchestrator/generated-skill" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1" + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill" + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill" + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill" + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator" + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest" + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml" + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field" + } + ], + "shared_resources": [] + }, + "stale_skills": [ + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "reason": "missing updated_at" + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill", + "reason": "review overdue by cadence monthly", + "age_days": 74, + "allowed_days": 31 + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "reason": "missing updated_at" + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "reason": "missing updated_at" + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "reason": "missing updated_at" + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "reason": "missing updated_at" + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "reason": "missing updated_at" + } + ], + "owner_review_gaps": [ + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "missing": [ + "owner", + "review_cadence" + ] + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + } + ], + "no_route_opportunities": [], + "artifacts": { + "catalog": "skill_atlas/catalog.json", + "route_overlap_matrix": "skill_atlas/route_overlap_matrix.csv", + "dependency_graph": "skill_atlas/dependency_graph.json", + "stale_skills": "skill_atlas/stale_skills.json", + "owner_review_gaps": "skill_atlas/owner_review_gaps.json", + "no_route_opportunities": "skill_atlas/no_route_opportunities.json", + "report_json": "reports/skill_atlas.json", + "report_html": "reports/skill_atlas.html" + } + }, + "manifest": { + "name": "yao-meta-skill", + "version": "1.1.0", + "owner": "Yao Team", + "updated_at": "2026-03-31", + "status": "active", + "maturity_tier": "governed", + "lifecycle_stage": "library", + "context_budget_tier": "production", + "review_cadence": "quarterly", + "target_platforms": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "factory_components": [ + "templates", + "references", + "scripts", + "reports" + ] + }, + "frontmatter": { + "name": "yao-meta-skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "metadata": { + "author": "Yao Team", + "philosophy": "structured design, evaluation loop, template ergonomics, operational packaging" + } + }, + "interface": { + "interface": { + "display_name": "Yao Meta Skill", + "short_description": "Create trigger-aware agent skills", + "default_prompt": "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." + }, + "compatibility": { + "canonical_format": "agent-skills", + "adapter_targets": [ + "openai", + "claude", + "generic" + ], + "activation": { + "mode": "manual", + "paths": [] + }, + "execution": { + "context": "inline", + "shell": "bash" + }, + "trust": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "degradation": { + "openai": "metadata-adapter", + "claude": "neutral-source-plus-adapter", + "generic": "neutral-source" + } + } + } + }, + "artifacts": { + "html": "reports/review-studio.html", + "json": "reports/review-studio.json" + } +} diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 8ecc4e58..aca4c7e3 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,15 +2,15 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 124, - "script_count": 49, + "scanned_files": 127, + "script_count": 50, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], "network_script_count": 2, "interactive_script_count": 0, - "package_sha256": "d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac" + "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" }, "failures": [], "warnings": [ @@ -275,6 +275,14 @@ "uses_network": false, "uses_subprocess": false }, + { + "path": "scripts/render_review_studio.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, { "path": "scripts/render_review_viewer.py", "has_argparse": true, diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 43089a59..00aa421c 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,12 +1,12 @@ # Security Trust Report - OK: `True` -- Scanned files: `124` -- Scripts: `49` +- Scanned files: `127` +- Scripts: `50` - Secret findings: `0` - Network-capable scripts: `2` - Interactive scripts: `0` -- Package SHA256: `d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac` +- Package SHA256: `18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad` ## Failures @@ -59,6 +59,7 @@ | scripts/render_reference_scan.py | True | True | False | False | False | | scripts/render_reference_synthesis.py | True | True | False | False | False | | scripts/render_regression_history.py | True | True | False | False | False | +| scripts/render_review_studio.py | True | True | False | False | False | | scripts/render_review_viewer.py | True | True | False | False | False | | scripts/render_skill_overview.py | True | True | False | False | False | | scripts/render_social_preview.py | False | True | False | False | False | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index ffa5f5b8..6fa359a0 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -13,8 +13,10 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Trust/Security v0 for secret, dependency, script, trust metadata, and package-integrity checks. - Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, and no-route opportunities. - Bilingual Skill Overview v2 that includes these evidence surfaces. +- Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, and release-gate review. +- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, parity checks, and IR provenance. -This is still not the final world-class state. Registry, compiler refactor, telemetry, and Review Studio 2.0 remain open. +This is still not the final world-class state. Registry, full target-specific compiler transforms, telemetry, and stricter governed trust gates remain open. ## Coverage Matrix @@ -25,31 +27,31 @@ This is still not the final world-class state. Registry, compiler refactor, tele | Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | -| Review Studio 2.0 | Current `reports/skill-overview.html`, `reports/review-viewer.html`, and `reports/skill_atlas.html` are separate | partial | +| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` | v0 landed | | Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `reports/skill_atlas.html` | v0 landed | -| Registry & Distribution | Existing packager, no registry audit/package schema yet | partial | +| Registry & Distribution | IR-sourced packager, no registry audit/package schema yet | partial | | Telemetry & Drift | Regression history exists, no adoption or activation telemetry yet | partial | -| Compiler from IR | Packager still reads source metadata directly | missing | +| Compiler from IR | Packager consumes Skill IR for core semantic adapter fields and keeps frontmatter/interface parity checks | v0 landed | ## Top Findings -### 1. Compiler path is not IR-first yet +### 1. Compiler path is IR-first v0, but transforms are still shallow -The new IR captures the capability contract, but `scripts/cross_packager.py` still builds target metadata directly from `SKILL.md` and `agents/interface.yaml`. +The packager now reads Skill IR for core semantic fields and emits provenance, schema version, job-to-be-done, semantic contract counts, governance, risk, targets, and semantic parity in each adapter. -Next move: add `scripts/compile_skill.py` or refactor packager to consume IR for core fields, then assert semantic parity. +Next move: add `scripts/compile_skill.py` or split the packager into target-specific transforms so OpenAI, Claude, Agent Skills, VS Code/Copilot, and generic packages can preserve runtime semantics beyond metadata. -### 2. Output eval is useful but too small +### 2. Output eval now meets the governed v0 minimum, but is still static -The v0 cases prove the runner and scorecard work, but the set is only three static cases. The 2.0 plan calls for library/governed coverage with at least five cases, including boundary, near-neighbor, and real file/output constraints. +The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. The next gap is that they are still static text comparisons rather than model-executed holdout runs with timing, tokens, and reviewer adjudication. -Next move: add holdout output cases and one file-backed fixture case. +Next move: add model-executed output eval runs, blind A/B comparison, and one real multi-file fixture. -### 3. Review Studio is still split +### 3. Review Studio is unified, but needs reviewer actions -The overview report is strong, but the reviewer still has to inspect multiple pages for output eval, conformance, trust, and release evidence. +The new Review Studio page aggregates intent, trigger, output, context, runtime, trust, atlas, and release gates. It now exposes current warnings directly: low generic intent-confidence context, trust-script warnings, and portfolio-level Atlas gaps. -Next move: add a Review Studio 2.0 section or page that aggregates trigger, output, conformance, trust, and release status into one blocking-issue surface. +Next move: add reviewer annotations, waived-risk records, and links from each warning to the exact source fix. ### 4. Multi-skill operation now has v0 coverage, but no telemetry @@ -67,17 +69,19 @@ Next move: add stricter governed-mode gates and package hash verification after | Gate | Current Result | | --- | --- | -| Output Eval | `3` cases, with-skill pass rate `100`, baseline pass rate `0` | +| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, and boundary coverage | | Runtime Conformance | `5 / 5` targets passing | | Trust | `0` secret findings, `1` pinned dependency file, `2` network-capable scripts flagged as warnings | | Skill Atlas | local workspace scan generated catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview | -| Context Budget | initial load remains under the production budget | +| Review Studio | `8` gates, `0` blockers, `3` warnings, world-class score `87/100` | +| IR-first Packaging | `openai`, `claude`, and `generic` adapters include IR provenance and semantic parity checks | +| Context Budget | initial load `910/1000`, under the production budget | | CI | `make ci-test` passed after adding v0 gates | ## Next Highest-Leverage Moves -1. Refactor packaging toward IR-first compiler behavior. -2. Add Review Studio 2.0 as a single reviewer surface. -3. Expand Output Eval Lab from static smoke cases to holdout and file-backed cases. +1. Split IR-first packaging into target-specific compiler transforms. +2. Add reviewer annotation and waived-risk records to Review Studio. +3. Expand Output Eval Lab from static cases to model-executed holdout and blind A/B cases. 4. Add registry package schema, package hash, and upgrade audit. 5. Connect Skill Atlas with telemetry and drift history. diff --git a/reports/skill-overview.html b/reports/skill-overview.html index ad83bfa5..6864237a 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -606,7 +606,7 @@ -
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
上下文成本42入口约 366 个词/字，references 约 11029 个词/字。入口约 366 个词/字，references 约 11029 个词/字。
+
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
上下文成本42入口约 346 个词/字，references 约 11313 个词/字。入口约 346 个词/字，references 约 11313 个词/字。
@@ -618,7 +618,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/skill-overview.html
+
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html
交付流程输入材料Skill 包体可复用能力
交付流程把用户输入、生成的包体和可复用能力放在一条线上。
@@ -638,10 +638,10 @@

指标判读Reading

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 366 个词/字。SKILL.md 约 366 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 366 个词/字，references 约 11029 个词/字。入口约 366 个词/字，references 约 11029 个词/字。
+
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 346 个词/字。SKILL.md 约 346 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 346 个词/字，references 约 11313 个词/字。入口约 346 个词/字，references 约 11313 个词/字。
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • intent-dialogue.json 已存在。intent-dialogue.json 已存在。
可维护性100
  • SKILL.md 约 366 个词/字。SKILL.md 约 366 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 366 个词/字，references 约 11029 个词/字。入口约 366 个词/字，references 约 11029 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • intent-dialogue.json 已存在。intent-dialogue.json 已存在。
可维护性100
  • SKILL.md 约 346 个词/字。SKILL.md 约 346 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 346 个词/字，references 约 11313 个词/字。入口约 346 个词/字，references 约 11313 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
@@ -717,7 +717,7 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 366 个词/字，references 约 11029 个词/字。上下文成本需要补强：入口约 366 个词/字，references 约 11029 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。纳入下一轮修复Fix next
@@ -734,7 +734,7 @@
- +
风险Risk信号Signal应对Response
误触发风险误触发风险frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
输出漂移风险输出漂移风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
证据不足风险证据不足风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
包体膨胀风险包体膨胀风险SKILL.md 约 366 个词/字。SKILL.md 约 366 个词/字。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
跨平台迁移风险跨平台迁移风险agents/interface.yaml 已存在。agents/interface.yaml 已存在。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
误触发风险误触发风险frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
输出漂移风险输出漂移风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
证据不足风险证据不足风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
包体膨胀风险包体膨胀风险SKILL.md 约 346 个词/字。SKILL.md 约 346 个词/字。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
跨平台迁移风险跨平台迁移风险agents/interface.yaml 已存在。agents/interface.yaml 已存在。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
@@ -748,7 +748,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布250项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布255项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 873d6459..47b39ada 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -16,6 +16,7 @@ "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/skill_atlas.html", + "reports/review-studio.html", "reports/skill-overview.html" ], "flow": [ @@ -61,7 +62,7 @@ "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 366 个词/字。", + "SKILL.md 约 346 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -82,7 +83,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 366 个词/字，references 约 11029 个词/字。", + "入口约 346 个词/字，references 约 11313 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -186,7 +187,7 @@ "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" ], "gaps": [ - "上下文成本需要补强：入口约 366 个词/字，references 约 11029 个词/字。" + "上下文成本需要补强：入口约 346 个词/字，references 约 11313 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -194,7 +195,7 @@ "每次升级后重新生成报告并检查分数原因。" ], "artifact_design": { - "design_system": "evidence frame", + "design_system": "review studio", "highlights": [ "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", "Name the working directory and required inputs for commands.", @@ -251,7 +252,7 @@ "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 366 个词/字。", + "signal": "SKILL.md 约 346 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -413,28 +414,28 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 28 + "file_count": 29 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 49 + "file_count": 50 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 27 + "file_count": 28 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 142 + "file_count": 144 } ], - "file_count": 250, + "file_count": 255, "folder_count": 4, "distribution": [ { @@ -455,19 +456,19 @@ }, { "label": "references", - "value": 28 + "value": 29 }, { "label": "scripts", - "value": 49 + "value": 50 }, { "label": "evals", - "value": 27 + "value": 28 }, { "label": "reports", - "value": 142 + "value": 144 } ] }, @@ -582,25 +583,25 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 28 + "file_count": 29 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 49 + "file_count": 50 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 27 + "file_count": 28 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 142 + "file_count": 144 } ], "strengths": [ @@ -704,10 +705,13 @@ "schema_version": "2.0.0", "target_count": 4, "trigger_samples": 8, - "output_eval_cases": 2 + "output_eval_cases": 3 }, "output_quality": { - "case_count": 3, + "case_count": 5, + "file_backed_case_count": 1, + "near_neighbor_case_count": 1, + "boundary_case_count": 1, "baseline_pass_rate": 0.0, "with_skill_pass_rate": 100.0, "delta": 100.0, @@ -721,15 +725,15 @@ "fail_count": 0 }, "trust_security": { - "scanned_files": 124, - "script_count": 49, + "scanned_files": 127, + "script_count": 50, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" ], "network_script_count": 2, "interactive_script_count": 0, - "package_sha256": "d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac" + "package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad" }, "skill_atlas": { "skill_count": 12, @@ -745,7 +749,7 @@ "Borrow the discipline of defining what the skill should not own before growing the package." ], "artifact_design": { - "design_system": "evidence frame", + "design_system": "review studio", "highlights": [ "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", "Name the working directory and required inputs for commands.", diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 17b5fdae..c6d1df26 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -62,6 +62,7 @@ "scripts/render_reference_scan.py", "scripts/render_reference_synthesis.py", "scripts/render_regression_history.py", + "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", @@ -100,6 +101,7 @@ "references/reference-scan.md", "references/regression-cause-taxonomy.md", "references/resource-boundaries.md", + "references/review-studio-method.md", "references/runtime-conformance-method.md", "references/skill-archetypes.md", "references/skill-atlas-method.md", diff --git a/scripts/build_skill_atlas.py b/scripts/build_skill_atlas.py index dc80bc37..fec8799f 100644 --- a/scripts/build_skill_atlas.py +++ b/scripts/build_skill_atlas.py @@ -91,7 +91,9 @@ def should_skip(path: Path, root: Path) -> bool: rel = path.relative_to(root) except ValueError: return True - return any(part in IGNORE_PARTS for part in rel.parts) + if any(part in IGNORE_PARTS for part in rel.parts): + return True + return len(rel.parts) >= 2 and rel.parts[0] == "tests" and rel.parts[1].startswith("tmp") def find_skill_dirs(workspace_root: Path) -> list[Path]: @@ -136,7 +138,8 @@ def resource_names(skill_dir: Path) -> list[str]: if not target.exists(): continue for path in sorted(target.rglob("*")): - if any(part in IGNORE_PARTS for part in path.relative_to(skill_dir).parts): + rel = path.relative_to(skill_dir) + if any(part in IGNORE_PARTS for part in rel.parts): continue if path.suffix in {".pyc", ".pyo"}: continue diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 52d5c79d..d01e1cf7 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -39,6 +39,7 @@ DEFAULT_TARGETS = [ "trust-check", "skill-atlas-check", "review-viewer-check", + "review-studio-check", "feedback-check", "baseline-compare-check", "reference-scan-check", diff --git a/scripts/cross_packager.py b/scripts/cross_packager.py index 2567edea..ae812163 100644 --- a/scripts/cross_packager.py +++ b/scripts/cross_packager.py @@ -7,6 +7,20 @@ from pathlib import Path import yaml +def display_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def read_json(path: Path) -> dict: + if not path.exists(): + return {} + payload = json.loads(path.read_text(encoding="utf-8")) + return payload if isinstance(payload, dict) else {} + + def read_simple_yaml(path: Path) -> dict: return yaml.safe_load(path.read_text(encoding="utf-8")) or {} @@ -31,6 +45,23 @@ def read_interface(skill_dir: Path) -> dict: return raw +def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict, str]: + candidates = [ + skill_dir / "reports" / "skill-ir.json", + skill_dir / "skill-ir" / "examples" / f"{name}.json", + skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json", + ] + seen = set() + for path in candidates: + if path in seen: + continue + seen.add(path) + payload = read_json(path) + if payload: + return payload, display_path(path, skill_dir) + return {}, "frontmatter-fallback" + + def require_fields(payload: dict, fields: list[str], label: str) -> None: missing = [field for field in fields if not payload.get(field)] if missing: @@ -46,8 +77,99 @@ def require_target_degradation( raise ValueError(f"Missing degradation entries for targets: {', '.join(missing)}") +def count_list(payload: dict, key: str) -> int: + value = payload.get(key, []) + return len(value) if isinstance(value, list) else 0 + + +def resource_counts(resources: dict) -> dict: + return { + "references": count_list(resources, "references"), + "scripts": count_list(resources, "scripts"), + "assets": count_list(resources, "assets"), + "reports": count_list(resources, "reports"), + } + + +def eval_counts(eval_plan: dict) -> dict: + return { + "trigger": count_list(eval_plan, "trigger"), + "output": count_list(eval_plan, "output"), + "adversarial": count_list(eval_plan, "adversarial"), + "baseline": 1 if eval_plan.get("baseline") else 0, + } + + +def build_semantic_contract( + *, + skill_dir: Path, + platform: str, + frontmatter: dict, + interface: dict, + compatibility: dict, + manifest: dict, + ir: dict, + ir_source: str, +) -> dict: + trigger_surface = ir.get("trigger_surface", {}) if isinstance(ir.get("trigger_surface"), dict) else {} + workflow = ir.get("workflow", {}) if isinstance(ir.get("workflow"), dict) else {} + resources = ir.get("resources", {}) if isinstance(ir.get("resources"), dict) else {} + eval_plan = ir.get("eval_plan", {}) if isinstance(ir.get("eval_plan"), dict) else {} + + frontmatter_name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name) + frontmatter_description = str(frontmatter.get("description") or "") + ir_name = str(ir.get("name") or "") if ir else "" + ir_description = str(trigger_surface.get("description") or "") if ir else "" + name = ir_name or frontmatter_name + description = ir_description or frontmatter_description + job = str(ir.get("job_to_be_done") or description) + targets = ir.get("targets") if isinstance(ir.get("targets"), list) else compatibility.get("adapter_targets", []) + target_values = [str(item) for item in targets] + adapter_targets = [str(item) for item in compatibility.get("adapter_targets", [])] + + semantic_contract = { + "name": name, + "description": description, + "job_to_be_done": job, + "trigger_description": description, + "should_trigger_count": count_list(trigger_surface, "should_trigger"), + "should_not_trigger_count": count_list(trigger_surface, "should_not_trigger"), + "edge_case_count": count_list(trigger_surface, "edge_cases"), + "workflow_step_count": count_list(workflow, "steps"), + "decision_point_count": count_list(workflow, "decision_points"), + "failure_mode_count": count_list(workflow, "failure_modes"), + "resource_counts": resource_counts(resources), + "eval_counts": eval_counts(eval_plan), + "risk": ir.get("risk", {}) if isinstance(ir.get("risk"), dict) else {}, + "governance": ir.get("governance", {}) if isinstance(ir.get("governance"), dict) else {}, + "targets": target_values, + "source_files_count": count_list(ir, "source_files") if ir else 0, + } + semantic_parity = { + "source": "skill-ir" if ir else "frontmatter-fallback", + "ir_source": ir_source, + "name_matches_ir": bool(ir) and frontmatter_name == name, + "description_matches_ir": bool(ir) and frontmatter_description == description, + "platform_declared_in_ir": platform in target_values + or (platform == "generic" and "agent-skills-compatible" in target_values), + "platform_declared_in_interface": platform in adapter_targets, + "display_name_present": bool(interface.get("display_name")), + "default_prompt_present": bool(interface.get("default_prompt")), + } + return { + "name": name, + "description": description, + "job_to_be_done": job, + "ir_source": ir_source, + "ir_schema_version": str(ir.get("schema_version") or "none"), + "semantic_contract": semantic_contract, + "semantic_parity": semantic_parity, + } + + def build_manifest(skill_dir: Path, platform: str) -> dict: frontmatter = read_frontmatter(skill_dir / "SKILL.md") + manifest = read_json(skill_dir / "manifest.json") interface_doc = read_interface(skill_dir) interface = interface_doc.get("interface", {}) compatibility = interface_doc.get("compatibility", {}) @@ -66,12 +188,28 @@ def build_manifest(skill_dir: Path, platform: str) -> dict: "compatibility.trust", ) require_target_degradation(degradation, compatibility.get("adapter_targets", [])) + ir, ir_source = find_skill_ir(skill_dir, str(frontmatter.get("name") or manifest.get("name") or skill_dir.name)) + semantic = build_semantic_contract( + skill_dir=skill_dir, + platform=platform, + frontmatter=frontmatter, + interface=interface, + compatibility=compatibility, + manifest=manifest, + ir=ir, + ir_source=ir_source, + ) return { - "name": frontmatter.get("name", skill_dir.name), - "description": frontmatter.get("description", ""), + "name": semantic["name"], + "description": semantic["description"], "version": frontmatter.get("version", "1.0.0"), "platform": platform, "skill_root": skill_dir.name, + "job_to_be_done": semantic["job_to_be_done"], + "ir_source": semantic["ir_source"], + "ir_schema_version": semantic["ir_schema_version"], + "semantic_contract": semantic["semantic_contract"], + "semantic_parity": semantic["semantic_parity"], "display_name": interface.get("display_name", skill_dir.name), "short_description": interface.get("short_description", ""), "default_prompt": interface.get("default_prompt", ""), @@ -108,6 +246,11 @@ PLATFORM_CONTRACTS = { "display_name", "short_description", "default_prompt", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity", "canonical_metadata", "canonical_format", "activation_mode", @@ -135,6 +278,11 @@ PLATFORM_CONTRACTS = { "display_name", "short_description", "default_prompt", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity", "canonical_metadata", "canonical_format", "activation_mode", @@ -162,6 +310,11 @@ PLATFORM_CONTRACTS = { "display_name", "short_description", "default_prompt", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity", "canonical_metadata", "canonical_format", "activation_mode", diff --git a/scripts/init_skill.py b/scripts/init_skill.py index 2abdaf99..0f59add5 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -15,6 +15,7 @@ from render_output_risk_profile import render_output_risk_profile from render_prompt_quality_profile import render_prompt_quality_profile from render_reference_scan import parse_reference, render_reference_scan from render_reference_synthesis import render_reference_synthesis +from render_review_studio import render_review_studio from render_review_viewer import render_review_viewer from render_skill_overview import render_skill_overview from render_system_model import render_system_model @@ -69,13 +70,14 @@ README_TEMPLATE = """# {title} 5. Open `reports/reference-synthesis.md` to see the GitHub benchmarks plus curated official, research, and principle tracks in one place. 6. Follow the workflow steps in `SKILL.md`. 7. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. It defaults to Simplified Chinese and includes an English switch in the top right. -8. Open `reports/review-viewer.html` for a compact visual review of the package. -9. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. -10. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. -11. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. -12. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. -13. Review `reports/iteration-directions.md` for the three most valuable next moves. -14. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. +8. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view. +9. Open `reports/review-viewer.html` for a compact visual review of the package. +10. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. +11. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. +12. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. +13. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. +14. Review `reports/iteration-directions.md` for the three most valuable next moves. +15. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. ## Honest Boundaries @@ -99,6 +101,7 @@ README_TEMPLATE = """# {title} - `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points - `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance - `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap +- `reports/review-studio.html`: Review Studio 2.0 gate page for intent, trigger, output eval, context, runtime conformance, trust, atlas, and release readiness - `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps - `reports/iteration-directions.md`: the top three next iteration directions """ @@ -211,14 +214,17 @@ def build_report_view(artifacts: dict) -> dict: html_report = artifacts.get("skill_overview_html", "") json_report = artifacts.get("skill_overview_json", "") system_model = artifacts.get("system_model_md", "") + review_studio = artifacts.get("review_studio_html", "") return { "title": "Skill 总结报告", "html_report": html_report, "json_report": json_report, "system_model": system_model, + "review_studio": review_studio, "message": ( f"Skill 已创建完成。建议先打开总结报告：{html_report}。" "它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线；" + f"然后打开 Review Studio 2.0：{review_studio}，检查意图、触发、输出评测、运行一致性、信任和发布闸门。" "报告默认使用中文简体，右上角可以切换英文版。" ), "next_action": "Open reports/skill-overview.html before editing more files.", @@ -243,6 +249,11 @@ def render_skill_ir(root: Path) -> dict: } +def absolute_skill_artifact(root: Path, value: str) -> str: + path = Path(value) + return str(path if path.is_absolute() else root / path) + + def dedupe_references(references: list[dict]) -> list[dict]: seen = set() deduped = [] @@ -328,6 +339,7 @@ def initialize_skill( iteration_directions = render_iteration_directions(root) overview = render_skill_overview(root) review_viewer = render_review_viewer(root) + review_studio = render_review_studio(root) artifacts = { "readme": str(root / "README.md"), "manifest": str(root / "manifest.json"), @@ -355,6 +367,8 @@ def initialize_skill( "iteration_directions_json": iteration_directions["artifacts"]["json"], "review_viewer_html": review_viewer["artifacts"]["html"], "review_viewer_json": review_viewer["artifacts"]["json"], + "review_studio_html": absolute_skill_artifact(root, review_studio["artifacts"]["html"]), + "review_studio_json": absolute_skill_artifact(root, review_studio["artifacts"]["json"]), } if benchmark_scan is not None: artifacts["github_benchmark_scan_md"] = benchmark_scan["artifacts"]["markdown"] diff --git a/scripts/render_portability_report.py b/scripts/render_portability_report.py index 00246df6..dbd8d23d 100644 --- a/scripts/render_portability_report.py +++ b/scripts/render_portability_report.py @@ -17,6 +17,17 @@ def load_yaml(path: Path) -> dict: return yaml.safe_load(path.read_text(encoding="utf-8")) or {} +def find_ir(root: Path) -> tuple[dict, str]: + candidates = [ + root / "reports" / "skill-ir.json", + root / "skill-ir" / "examples" / f"{root.name}.json", + ] + for path in candidates: + if path.exists(): + return load_json(path), str(path.relative_to(root)) + return {}, "missing" + + def band_for(score: int) -> str: if score >= 97: return "world-class" @@ -39,9 +50,12 @@ def build_report(root: Path) -> dict: contracts_doc = (root / "references" / "packaging-contracts.md").exists() matrix_doc = (root / "references" / "platform-capability-matrix.md").exists() snapshot_files = list((root / "tests" / "snapshots").glob("*_adapter.json")) + ir, ir_source = find_ir(root) + has_neutral_metadata = (root / "SKILL.md").exists() and (root / "agents" / "interface.yaml").exists() + has_skill_ir = ir.get("schema_version") == "2.0.0" and bool(ir.get("job_to_be_done")) breakdown = { - "canonical_neutral_source": 15 if (root / "SKILL.md").exists() and (root / "agents" / "interface.yaml").exists() else 0, + "platform_neutral_semantic_source": 15 if has_neutral_metadata and has_skill_ir else 8 if has_neutral_metadata else 0, "adapter_target_coverage": 15 if len(targets) >= 3 else 10 if len(targets) >= 2 else 5 if len(targets) >= 1 else 0, "activation_metadata": 10 if activation.get("mode") and activation.get("paths") is not None @@ -83,6 +97,8 @@ def build_report(root: Path) -> dict: "remote_inline_execution": trust.get("remote_inline_execution"), "degradation_coverage": sum(1 for target in targets if degradation.get(target)), "snapshot_count": len(snapshot_files), + "ir_source": ir_source, + "ir_schema_version": ir.get("schema_version", "missing"), }, "breakdown": breakdown, } @@ -100,6 +116,8 @@ def render_markdown(report: dict) -> str: f"- shell: `{report['summary']['shell']}`", f"- trust level: `{report['summary']['trust_level']}`", f"- remote inline execution: `{report['summary']['remote_inline_execution']}`", + f"- Skill IR source: `{report['summary']['ir_source']}`", + f"- Skill IR schema: `{report['summary']['ir_schema_version']}`", "", "| Component | Score |", "| --- | ---: |", diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py new file mode 100644 index 00000000..6ce6f777 --- /dev/null +++ b/scripts/render_review_studio.py @@ -0,0 +1,720 @@ +#!/usr/bin/env python3 +import argparse +import html +import json +import os +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def parse_frontmatter(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + if not lines or lines[0].strip() != "---": + return {} + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {} + text = "\n".join(lines[1:end_index]) + if yaml is not None: + payload = yaml.safe_load(text) or {} + return payload if isinstance(payload, dict) else {} + data = {} + for line in text.splitlines(): + if ":" in line: + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data + + +def display_path(skill_dir: Path, path: Path) -> str: + try: + return str(path.resolve().relative_to(skill_dir.resolve())) + except ValueError: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def link_from(output_html: Path, target: Path) -> str: + return os.path.relpath(target.resolve(), output_html.parent.resolve()) + + +def report_link(output_html: Path, skill_dir: Path, rel_path: str) -> str: + return link_from(output_html, skill_dir / rel_path) + + +def gate(key: str, label: str, status: str, detail: str, evidence: str, link: str = "") -> dict[str, str]: + return { + "key": key, + "label": label, + "status": status, + "detail": detail, + "evidence": evidence, + "link": link, + } + + +def status_label(status: str) -> str: + return {"pass": "通过", "warn": "关注", "block": "阻断"}.get(status, status) + + +def add_blockers_from_gate(gates: list[dict[str, str]]) -> tuple[list[dict[str, str]], list[dict[str, str]]]: + blockers = [item for item in gates if item["status"] == "block"] + warnings = [item for item in gates if item["status"] == "warn"] + return blockers, warnings + + +def target_maturity(skill_dir: Path, overview: dict[str, Any]) -> str: + manifest = load_json(skill_dir / "manifest.json") + if manifest.get("maturity_tier"): + return str(manifest["maturity_tier"]) + metadata = overview.get("metadata", {}) if isinstance(overview, dict) else {} + if metadata.get("maturity_tier"): + return str(metadata["maturity_tier"]) + return "scaffold" + + +def min_output_cases(maturity: str) -> int: + if maturity in {"library", "governed"}: + return 5 + if maturity == "production": + return 3 + return 1 + + +def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: + overview = data["overview"] + maturity = target_maturity(skill_dir, overview) + gates: list[dict[str, str]] = [] + + intent = data["intent_confidence"] + intent_score = int(intent.get("score", 0) or 0) + intent_status = "pass" if intent.get("gate_passed") or intent_score >= 75 else "warn" + gates.append( + gate( + "intent-canvas", + "意图画布", + intent_status, + f"intent confidence {intent_score}/100; {intent.get('recommended_action', 'review current intent frame')}", + "reports/intent-confidence.json", + report_link(output_html, skill_dir, "reports/intent-confidence.md"), + ) + ) + + route = data["route_scorecard"] + route_summary = route.get("summary", {}) + misroutes = int(route_summary.get("misroute_count", len(route.get("misroutes", []))) or 0) + ambiguous = int(route_summary.get("ambiguous_case_count", len(route.get("ambiguous_cases", []))) or 0) + if not route: + route_status = "warn" + route_detail = "route scorecard is missing; run route-scorecard before release review" + else: + route_status = "block" if misroutes else ("warn" if ambiguous else "pass") + route_detail = f"{route_summary.get('total_cases', 0)} trigger cases; {misroutes} misroutes; {ambiguous} ambiguous" + gates.append( + gate( + "trigger-lab", + "触发实验", + route_status, + route_detail, + "reports/route_scorecard.json", + report_link(output_html, skill_dir, "reports/route_scorecard.md"), + ) + ) + + output = data["output_quality"] + output_summary = output.get("summary", {}) + required_cases = min_output_cases(maturity) + case_count = int(output_summary.get("case_count", 0) or 0) + file_backed = int(output_summary.get("file_backed_case_count", 0) or 0) + near_neighbor = int(output_summary.get("near_neighbor_case_count", 0) or 0) + boundary = int(output_summary.get("boundary_case_count", 0) or 0) + output_blocked = not output.get("ok", False) or not output_summary.get("gate_pass", False) or case_count < required_cases + output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 + if not output: + output_status = "warn" + output_detail = "output eval scorecard is missing; generate it before production review" + else: + output_status = "block" if output_blocked else ("warn" if output_warn else "pass") + output_detail = ( + f"{case_count}/{required_cases} cases; with-skill {output_summary.get('with_skill_pass_rate', 0)}; " + f"baseline {output_summary.get('baseline_pass_rate', 0)}; file-backed {file_backed}; near-neighbor {near_neighbor}" + ) + gates.append( + gate( + "output-lab", + "输出实验", + output_status, + output_detail, + "reports/output_quality_scorecard.json", + report_link(output_html, skill_dir, "reports/output_quality_scorecard.md"), + ) + ) + + context = data["context_budget"] + context_stats = context.get("stats", {}) + context_status = "pass" if context.get("ok") else "block" + if context.get("warnings"): + context_status = "warn" if context_status == "pass" else context_status + if not context: + context_status = "warn" + context_detail = ( + f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/" + f"{context_stats.get('context_budget_limit', 'n/a')}; quality density {context_stats.get('quality_density', 'n/a')}" + ) + gates.append( + gate( + "context-budget", + "上下文", + context_status, + context_detail, + "reports/context_budget.json", + report_link(output_html, skill_dir, "reports/context_budget.md"), + ) + ) + + conformance = data["conformance"] + conformance_summary = conformance.get("summary", {}) + fail_count = int(conformance_summary.get("fail_count", 0) or 0) + if not conformance: + conformance_status = "warn" + conformance_detail = "runtime conformance matrix is missing" + else: + conformance_status = "block" if fail_count else "pass" + conformance_detail = f"{conformance_summary.get('pass_count', 0)} / {conformance_summary.get('target_count', 0)} targets pass" + gates.append( + gate( + "runtime-matrix", + "运行矩阵", + conformance_status, + conformance_detail, + "reports/conformance_matrix.json", + report_link(output_html, skill_dir, "reports/conformance_matrix.md"), + ) + ) + + trust = data["trust"] + trust_summary = trust.get("summary", {}) + if not trust: + trust_status = "warn" + trust_detail = "security trust report is missing" + else: + trust_status = "block" if trust.get("failures") else ("warn" if trust.get("warnings") else "pass") + trust_detail = ( + f"{trust_summary.get('secret_findings', 0)} secrets; " + f"{trust_summary.get('script_count', 0)} scripts; " + f"{trust_summary.get('network_script_count', 0)} network-capable scripts" + ) + gates.append( + gate( + "trust-report", + "信任报告", + trust_status, + trust_detail, + "reports/security_trust_report.json", + report_link(output_html, skill_dir, "reports/security_trust_report.md"), + ) + ) + + atlas = data["atlas"] + atlas_summary = atlas.get("summary", {}) + atlas_issues = int(atlas_summary.get("route_collision_count", 0) or 0) + int(atlas_summary.get("owner_gap_count", 0) or 0) + if not atlas: + atlas_status = "warn" + atlas_detail = "skill atlas is missing; portfolio-level conflicts are unknown" + else: + atlas_status = "warn" if atlas_issues else "pass" + atlas_detail = ( + f"{atlas_summary.get('skill_count', 0)} skills; " + f"{atlas_summary.get('route_collision_count', 0)} route collisions; " + f"{atlas_summary.get('owner_gap_count', 0)} owner gaps; " + f"{atlas_summary.get('stale_count', 0)} stale" + ) + gates.append( + gate( + "skill-atlas", + "组合治理", + atlas_status, + atlas_detail, + "reports/skill_atlas.json", + report_link(output_html, skill_dir, "reports/skill_atlas.html"), + ) + ) + + promotion = data["promotion"] + migration_path = ROOT / "docs" / "migration-v2.md" + if promotion: + promotion_summary = promotion.get("summary", {}) + blocked = int(promotion_summary.get("blocked", 0) or 0) + release_status = "block" if blocked else "pass" + release_detail = f"{promotion_summary.get('promote', 0)} promote; {promotion_summary.get('keep_current', 0)} keep current; {blocked} blocked" + else: + release_status = "warn" + release_detail = "promotion decisions are missing; release notes need reviewer confirmation" + gates.append( + gate( + "release-notes", + "发布路线", + release_status, + release_detail, + "reports/promotion_decisions.json + docs/migration-v2.md", + report_link(output_html, skill_dir, "reports/promotion_decisions.md") if promotion else str(migration_path), + ) + ) + + return gates + + +def weighted_score(gates: list[dict[str, str]]) -> int: + weights = { + "trigger-lab": 15, + "output-lab": 20, + "context-budget": 10, + "runtime-matrix": 10, + "trust-report": 10, + "skill-atlas": 10, + "release-notes": 10, + "intent-canvas": 10, + } + earned = 0.0 + total = 0.0 + for item in gates: + weight = weights.get(item["key"], 5) + total += weight + if item["status"] == "pass": + earned += weight + elif item["status"] == "warn": + earned += weight * 0.6 + return int(round(earned / total * 100)) if total else 0 + + +def evidence_paths(skill_dir: Path) -> dict[str, str]: + rels = { + "skill_overview": "reports/skill-overview.html", + "review_viewer": "reports/review-viewer.html", + "output_eval": "reports/output_quality_scorecard.md", + "runtime_conformance": "reports/conformance_matrix.md", + "trust_report": "reports/security_trust_report.md", + "skill_atlas": "reports/skill_atlas.html", + "migration": "docs/migration-v2.md", + "skill_ir": "reports/skill-ir.json", + } + return {key: rel for key, rel in rels.items() if (skill_dir / rel).exists() or (ROOT / rel).exists()} + + +def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: + reports = skill_dir / "reports" + return { + "overview": load_json(reports / "skill-overview.json"), + "intent_confidence": load_json(reports / "intent-confidence.json"), + "intent_dialogue": load_json(reports / "intent-dialogue.json"), + "route_scorecard": load_json(reports / "route_scorecard.json"), + "output_quality": load_json(reports / "output_quality_scorecard.json"), + "conformance": load_json(reports / "conformance_matrix.json"), + "trust": load_json(reports / "security_trust_report.json"), + "context_budget": load_json(reports / "context_budget.json"), + "promotion": load_json(reports / "promotion_decisions.json"), + "atlas": load_json(reports / "skill_atlas.json"), + "manifest": load_json(skill_dir / "manifest.json"), + "frontmatter": parse_frontmatter(skill_dir / "SKILL.md"), + "interface": load_yaml(skill_dir / "agents" / "interface.yaml"), + } + + +def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: + overview = data["overview"] + output = data["output_quality"].get("summary", {}) + conformance = data["conformance"].get("summary", {}) + trust = data["trust"].get("summary", {}) + atlas = data["atlas"].get("summary", {}) + cards = [ + { + "label": "Skill IR", + "value": str(overview.get("skill_ir", {}).get("schema_version", "missing")), + "detail": f"{overview.get('skill_ir', {}).get('target_count', 0)} targets in platform-neutral contract", + }, + { + "label": "Output Delta", + "value": str(output.get("delta", "n/a")), + "detail": f"{output.get('case_count', 0)} cases; {output.get('file_backed_case_count', 0)} file-backed", + }, + { + "label": "Runtime", + "value": f"{conformance.get('pass_count', 0)}/{conformance.get('target_count', 0)}", + "detail": "target conformance pass rate", + }, + { + "label": "Trust", + "value": str(trust.get("secret_findings", 0)), + "detail": f"{trust.get('script_count', 0)} scripts scanned; secrets found", + }, + { + "label": "Atlas", + "value": str(atlas.get("route_collision_count", 0)), + "detail": f"{atlas.get('skill_count', 0)} scanned skills; route collisions", + }, + ] + return cards + + +def render_gate_list(gates: list[dict[str, str]]) -> str: + items = [] + for item in gates: + link_html = f"证据" if item.get("link") else "" + items.append( + "
" + f"
{html.escape(status_label(item['status']))}

{html.escape(item['label'])}

" + f"

{html.escape(item['detail'])}

" + f"
{html.escape(item['evidence'])} {link_html}
" + "
" + ) + return "".join(items) + + +def render_insights(cards: list[dict[str, str]]) -> str: + return "".join( + ( + "
" + f"{html.escape(item['label'])}" + f"{html.escape(item['value'])}" + f"

{html.escape(item['detail'])}

" + "
" + ) + for item in cards + ) + + +def render_issue_list(title: str, items: list[dict[str, str]]) -> str: + if not items: + return f"

{html.escape(title)}

无。

" + body = "".join( + ( + "
  • " + f"{html.escape(item['label'])}" + f"{html.escape(item['detail'])}" + "
    • " + ) + for item in items + ) + return f"

    {html.escape(title)}

      {body}
    " + + +def render_html(report: dict[str, Any]) -> str: + summary = report["summary"] + gates = report["gates"] + blockers = report["blockers"] + warnings = report["warnings"] + insights = insight_cards(report["data"]) + overview = report["data"]["overview"] + manifest = report["data"]["manifest"] + frontmatter = report["data"]["frontmatter"] + title = overview.get("display_name") or overview.get("title") or frontmatter.get("name") or manifest.get("name") or "Skill" + description = overview.get("description") or frontmatter.get("description", "") + nav = [ + ("#overview", "审查总览"), + ("#intent", "意图画布"), + ("#trigger", "触发实验"), + ("#output", "输出实验"), + ("#runtime", "运行矩阵"), + ("#trust", "信任报告"), + ("#atlas", "组合治理"), + ("#release", "发布路线"), + ] + nav_html = "".join(f"{label}" for href, label in nav) + gates_html = render_gate_list(gates) + metrics_html = render_insights(insights) + blockers_html = render_issue_list("阻断事项", blockers) + warnings_html = render_issue_list("关注事项", warnings) + output_summary = report["data"]["output_quality"].get("summary", {}) + conformance_summary = report["data"]["conformance"].get("summary", {}) + trust_summary = report["data"]["trust"].get("summary", {}) + atlas_summary = report["data"]["atlas"].get("summary", {}) + evidence_html = "".join( + f"
  • {html.escape(key)}{html.escape(value)}
    • " + for key, value in report["evidence_paths"].items() + ) + return f""" + + + + + {html.escape(str(title))} Review Studio 2.0 + + + + +
    +
    +
    Review Studio 2.0
    +

    {html.escape(str(title))}

    +

    {html.escape(str(description))}

    +
    + 审查结论 + {html.escape(summary['decision'])} + Score {html.escape(str(summary['world_class_score']))}/100 +
    +
    + +
    +

    核心指标

    +
    {metrics_html}
    +
    + +
    +

    审查闸门

    +
    {gates_html}
    +
    + +
    + {blockers_html} + {warnings_html} +
    + +
    +
    +

    意图画布

    +

    {html.escape(str(report['data']['intent_confidence'].get('anchor_sentence', description)))}

    +
    +
    +

    证据路径

    +
      {evidence_html}
    +
    +
    + +
    +

    触发实验

    {html.escape(gates[1]['detail'])}

    +

    组合治理

    {html.escape(str(atlas_summary))}

    +
    + +
    +

    输出实验

    {html.escape(str(output_summary))}

    +

    发布标准

    Governed 和 Library 至少需要 5 个 output eval cases，并覆盖 file-backed、near-neighbor 和 boundary case。

    +
    + +
    +

    运行矩阵

    {html.escape(str(conformance_summary))}

    +

    上下文

    {html.escape(gates[3]['detail'])}

    +
    + +
    +

    信任报告

    {html.escape(str(trust_summary))}

    +

    安全边界

    高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

    +
    + +
    +

    组合治理

    {html.escape(gates[6]['detail'])}

    +

    下一动作

    优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps，再做 registry 或 telemetry。

    +
    + +
    +

    发布路线

    {html.escape(gates[7]['detail'])}

    +

    世界级缺口

    下一阶段应继续推进 IR-first compiler、registry audit、telemetry drift loop，以及更严格的 governed trust gates。

    +
    +
    + + +""" + + +def render_review_studio(skill_dir: Path, output_html: Path | None = None, output_json: Path | None = None) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + output_html = output_html or reports_dir / "review-studio.html" + output_json = output_json or reports_dir / "review-studio.json" + data = load_review_data(skill_dir) + gates = build_gates(skill_dir, output_html, data) + blockers, warnings = add_blockers_from_gate(gates) + score = weighted_score(gates) + decision = "blocked" if blockers else ("review" if warnings else "ready") + report = { + "schema_version": "2.0", + "ok": True, + "skill_dir": display_path(skill_dir, skill_dir), + "summary": { + "decision": decision, + "world_class_score": score, + "gate_count": len(gates), + "blocker_count": len(blockers), + "warning_count": len(warnings), + }, + "gates": gates, + "blockers": blockers, + "warnings": warnings, + "evidence_paths": evidence_paths(skill_dir), + "data": data, + "artifacts": { + "html": display_path(skill_dir, output_html), + "json": display_path(skill_dir, output_json), + }, + } + output_html.write_text(render_html(report), encoding="utf-8") + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return {key: value for key, value in report.items() if key != "data"} + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render Review Studio 2.0 for a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-html") + parser.add_argument("--output-json") + args = parser.parse_args() + payload = render_review_studio( + Path(args.skill_dir), + output_html=Path(args.output_html).resolve() if args.output_html else None, + output_json=Path(args.output_json).resolve() if args.output_json else None, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_output_eval.py b/scripts/run_output_eval.py index 2106931e..8d79f45f 100644 --- a/scripts/run_output_eval.py +++ b/scripts/run_output_eval.py @@ -36,11 +36,24 @@ def normalize(text: str) -> str: return str(text).casefold() -def validate_case(case: dict[str, Any]) -> list[str]: +def validate_case(case: dict[str, Any], cases_root: Path) -> list[str]: failures = [] for key in ("id", "prompt", "baseline_output", "with_skill_output", "assertions"): if key not in case: failures.append(f"{case.get('id', '')}: missing {key}") + for raw_path in case.get("input_files", []): + rel = Path(str(raw_path)) + if rel.is_absolute(): + failures.append(f"{case.get('id', '')}: input_files must be relative: {raw_path}") + continue + target = (cases_root / rel).resolve() + try: + target.relative_to(cases_root.resolve()) + except ValueError: + failures.append(f"{case.get('id', '')}: input_file escapes eval folder: {raw_path}") + continue + if not target.exists(): + failures.append(f"{case.get('id', '')}: input_file is missing: {raw_path}") assertions = case.get("assertions", []) if not isinstance(assertions, list) or not assertions: failures.append(f"{case.get('id', '')}: assertions must be a non-empty list") @@ -92,6 +105,8 @@ def grade_case(case: dict[str, Any]) -> dict[str, Any]: return { "id": case["id"], "prompt": case["prompt"], + "input_files": case.get("input_files", []), + "metadata": case.get("metadata", {}), "baseline": baseline, "with_skill": with_skill, "delta": round(with_skill["score"] - baseline["score"], 2), @@ -106,8 +121,14 @@ def build_summary(results: list[dict[str, Any]]) -> dict[str, Any]: with_skill_average = sum(item["with_skill"]["score"] for item in results) / case_count if case_count else 0 regressions = [item for item in results if item["delta"] < 0] failures = sorted({failure for item in results for failure in item["failure_taxonomy"]}) + file_backed = [item for item in results if item.get("input_files")] + near_neighbors = [item for item in results if item.get("metadata", {}).get("case_type") == "near_neighbor"] + boundary_cases = [item for item in results if item.get("metadata", {}).get("case_type") == "boundary"] return { "case_count": case_count, + "file_backed_case_count": len(file_backed), + "near_neighbor_case_count": len(near_neighbors), + "boundary_case_count": len(boundary_cases), "baseline_pass_rate": round(baseline_average, 2), "with_skill_pass_rate": round(with_skill_average, 2), "delta": round(with_skill_average - baseline_average, 2), @@ -162,7 +183,7 @@ def render_markdown(payload: dict[str, Any]) -> str: def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dict[str, Any]: cases = load_cases(cases_path) - validation_failures = [failure for case in cases for failure in validate_case(case)] + validation_failures = [failure for case in cases for failure in validate_case(case, cases_path.parent)] if validation_failures: payload = { "ok": False, diff --git a/scripts/skill_report_model.py b/scripts/skill_report_model.py index f27f0e28..afdb36d0 100644 --- a/scripts/skill_report_model.py +++ b/scripts/skill_report_model.py @@ -202,6 +202,8 @@ def derive_strengths(skill_dir: Path, metadata: dict) -> list[str]: strengths.append("已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。") if (skill_dir / "reports" / "skill_atlas.json").exists(): strengths.append("已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。") + if (skill_dir / "reports" / "review-studio.json").exists(): + strengths.append("已生成 Review Studio 2.0，可在一页中查看 blocker、warning、证据路径和发布闸门。") if (skill_dir / "agents" / "interface.yaml").exists(): strengths.append("已打包 agents/interface.yaml，便于后续做跨平台适配。") if (skill_dir / "references").exists() and any((skill_dir / "references").iterdir()): @@ -500,6 +502,7 @@ def build_report_model(skill_dir: Path) -> dict: "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/skill_atlas.html", + "reports/review-studio.html", "reports/skill-overview.html", ], "flow": ["输入材料", "Skill 包体", "可复用能力"], diff --git a/scripts/yao.py b/scripts/yao.py index 1f569580..61bc7ba5 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -509,6 +509,7 @@ def command_quickstart(args: argparse.Namespace) -> int: next_steps = [ "Open reports/skill-overview.html to review the generated Skill audit report.", "Open reports/intent-dialogue.md and tighten the real job, outputs, and exclusions.", + "Open reports/review-studio.html to inspect the Review Studio 2.0 gate view before release.", "Open reports/review-viewer.html to explain the package to a first-time reviewer.", "Use reports/iteration-directions.md to choose only one high-value next move before adding more files.", ] @@ -543,6 +544,7 @@ def command_quickstart(args: argparse.Namespace) -> int: "artifact_design_profile": payload.get("artifacts", {}).get("artifact_design_profile_md"), "prompt_quality_profile": payload.get("artifacts", {}).get("prompt_quality_profile_md"), "system_model": payload.get("artifacts", {}).get("system_model_md"), + "review_studio": payload.get("artifacts", {}).get("review_studio_html"), "review_viewer": payload.get("artifacts", {}).get("review_viewer_html"), }, }, @@ -707,6 +709,18 @@ def command_review_viewer(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_review_studio(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.output_html: + cmd.extend(["--output-html", args.output_html]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + result = run_script("render_review_studio.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_reference_scan(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -1196,6 +1210,12 @@ def build_parser() -> argparse.ArgumentParser: review_viewer_cmd.add_argument("--output-json") review_viewer_cmd.set_defaults(func=command_review_viewer) + review_studio_cmd = subparsers.add_parser("review-studio", help="Render Review Studio 2.0 for a skill package.") + review_studio_cmd.add_argument("skill_dir", nargs="?", default=".") + review_studio_cmd.add_argument("--output-html") + review_studio_cmd.add_argument("--output-json") + review_studio_cmd.set_defaults(func=command_review_studio) + reference_scan_cmd = subparsers.add_parser( "reference-scan", help="Render a controlled benchmark scan report for a skill package.", diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index 549bd743..6ee85552 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -55,10 +55,9 @@ "`Scaffold`: exploratory or personal.", "`Production`: team reuse.", "`Library`: shared infrastructure.", - "a working skill directory", - "a `SKILL.md`", - "aligned `agents/interface.yaml`", - "optional `references/`, `scripts/`, `evals/`, `reports/`, and `manifest.json` when justified" + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." ], "failure_modes": [ "Tables can render as dense grids with weak hierarchy or poor mobile readability.", @@ -96,6 +95,7 @@ "references/reference-scan.md", "references/regression-cause-taxonomy.md", "references/resource-boundaries.md", + "references/review-studio-method.md", "references/runtime-conformance-method.md", "references/skill-archetypes.md", "references/skill-atlas-method.md", @@ -137,6 +137,7 @@ "scripts/render_reference_scan.py", "scripts/render_reference_synthesis.py", "scripts/render_regression_history.py", + "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", @@ -213,6 +214,7 @@ "evals/history/description_optimization/README.md", "evals/holdout/trigger_cases.json", "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", "evals/output/schema.json", "evals/packaging_expectations.json", "evals/promotion_policy.md", @@ -222,6 +224,7 @@ ], "output": [ "evals/output/cases.jsonl", + "evals/output/fixtures/release-brief-source.md", "evals/output/schema.json" ], "adversarial": [ diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index db86f898..1533c9ff 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -51,6 +51,7 @@ "scripts/render_reference_scan.py", "scripts/render_reference_synthesis.py", "scripts/render_regression_history.py", + "scripts/render_review_studio.py", "scripts/render_review_viewer.py", "scripts/render_skill_overview.py", "scripts/render_social_preview.py", @@ -89,6 +90,7 @@ "references/reference-scan.md", "references/regression-cause-taxonomy.md", "references/resource-boundaries.md", + "references/review-studio-method.md", "references/runtime-conformance-method.md", "references/skill-archetypes.md", "references/skill-atlas-method.md", diff --git a/tests/snapshots/claude_adapter.json b/tests/snapshots/claude_adapter.json index dc0757fd..35e4975e 100644 --- a/tests/snapshots/claude_adapter.json +++ b/tests/snapshots/claude_adapter.json @@ -10,6 +10,11 @@ "trust_level", "remote_inline_execution", "degradation_strategy", - "portability_profile" + "portability_profile", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity" ] } diff --git a/tests/snapshots/generic_adapter.json b/tests/snapshots/generic_adapter.json index 83e5dae9..cb976c50 100644 --- a/tests/snapshots/generic_adapter.json +++ b/tests/snapshots/generic_adapter.json @@ -10,6 +10,11 @@ "trust_level", "remote_inline_execution", "degradation_strategy", - "portability_profile" + "portability_profile", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity" ] } diff --git a/tests/snapshots/openai_adapter.json b/tests/snapshots/openai_adapter.json index 6bc94997..3a3180ec 100644 --- a/tests/snapshots/openai_adapter.json +++ b/tests/snapshots/openai_adapter.json @@ -10,6 +10,11 @@ "trust_level", "remote_inline_execution", "degradation_strategy", - "portability_profile" + "portability_profile", + "job_to_be_done", + "ir_source", + "ir_schema_version", + "semantic_contract", + "semantic_parity" ] } diff --git a/tests/verify_adapter_snapshots.py b/tests/verify_adapter_snapshots.py index 51cb9104..a8d3ea7a 100644 --- a/tests/verify_adapter_snapshots.py +++ b/tests/verify_adapter_snapshots.py @@ -53,6 +53,28 @@ def main() -> None: for field in snapshot.get("required_fields", []): if field not in adapter: failures.append(f"{name}: missing required adapter field {field}") + if adapter.get("ir_source") != "skill-ir/examples/yao-meta-skill.json": + failures.append(f"{name}: adapter is not sourced from root Skill IR") + if adapter.get("ir_schema_version") != "2.0.0": + failures.append(f"{name}: missing Skill IR schema version") + contract = adapter.get("semantic_contract", {}) + if contract.get("name") != "yao-meta-skill": + failures.append(f"{name}: semantic contract name mismatch") + if contract.get("trigger_description") != adapter.get("description"): + failures.append(f"{name}: trigger description is not adapter description") + if contract.get("job_to_be_done") != adapter.get("job_to_be_done"): + failures.append(f"{name}: job-to-be-done not carried into adapter") + if contract.get("resource_counts", {}).get("references", 0) <= 0: + failures.append(f"{name}: semantic contract does not include reference counts") + if contract.get("eval_counts", {}).get("output", 0) <= 0: + failures.append(f"{name}: semantic contract does not include output eval counts") + parity = adapter.get("semantic_parity", {}) + if parity.get("source") != "skill-ir": + failures.append(f"{name}: semantic parity source is not skill-ir") + if parity.get("name_matches_ir") is not True: + failures.append(f"{name}: frontmatter name does not match Skill IR") + if parity.get("description_matches_ir") is not True: + failures.append(f"{name}: frontmatter description does not match Skill IR") if not (TMP / snapshot["required_generated_file"]).exists(): failures.append(f"{name}: missing generated file {snapshot['required_generated_file']}") if name == "openai": diff --git a/tests/verify_output_eval_lab.py b/tests/verify_output_eval_lab.py index 71661b23..15c5d2e1 100644 --- a/tests/verify_output_eval_lab.py +++ b/tests/verify_output_eval_lab.py @@ -32,7 +32,10 @@ def main() -> None: ) payload = json.loads(proc.stdout) assert payload["ok"], payload - assert payload["summary"]["case_count"] == 3, payload + assert payload["summary"]["case_count"] == 5, payload + assert payload["summary"]["file_backed_case_count"] == 1, payload + assert payload["summary"]["near_neighbor_case_count"] == 1, payload + assert payload["summary"]["boundary_case_count"] == 1, payload assert payload["summary"]["with_skill_pass_rate"] > payload["summary"]["baseline_pass_rate"], payload assert payload["summary"]["delta"] > 0, payload assert payload["summary"]["gate_pass"], payload diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py new file mode 100644 index 00000000..f4473f17 --- /dev/null +++ b/tests/verify_review_studio.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_review_studio.py" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_review_studio" + tmp_root.mkdir(parents=True, exist_ok=True) + subprocess.run([sys.executable, str(ROOT / "scripts" / "run_output_eval.py")], cwd=ROOT, check=True, capture_output=True, text=True) + + output_html = tmp_root / "review-studio.html" + output_json = tmp_root / "review-studio.json" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-html", + str(output_html), + "--output-json", + str(output_json), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["ok"], payload + assert payload["schema_version"] == "2.0", payload + assert payload["summary"]["gate_count"] == 8, payload + assert payload["summary"]["world_class_score"] > 0, payload + gate_keys = {item["key"] for item in payload["gates"]} + assert {"intent-canvas", "trigger-lab", "output-lab", "runtime-matrix", "trust-report", "skill-atlas", "release-notes"} <= gate_keys, payload + output_gate = next(item for item in payload["gates"] if item["key"] == "output-lab") + assert output_gate["status"] == "pass", output_gate + assert "5/5 cases" in output_gate["detail"], output_gate + assert "file-backed 1" in output_gate["detail"], output_gate + assert output_html.exists(), output_html + assert output_json.exists(), output_json + html = output_html.read_text(encoding="utf-8") + assert "Review Studio 2.0" in html, html[:400] + assert "审查闸门" in html, html[:1200] + assert "输出实验" in html, html[:2000] + assert str(ROOT) not in output_json.read_text(encoding="utf-8"), output_json + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_overview.py b/tests/verify_skill_overview.py index da16ff35..811fe65c 100644 --- a/tests/verify_skill_overview.py +++ b/tests/verify_skill_overview.py @@ -83,6 +83,7 @@ def main() -> None: assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/skill_atlas.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/review-studio.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert overview_json["skill_ir"]["schema_version"] in {"", "2.0.0"}, overview_json.get("skill_ir") assert "output_quality" in overview_json, overview_json.keys() assert "runtime_conformance" in overview_json, overview_json.keys() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 1673b286..9ff14b33 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -45,6 +45,8 @@ def main() -> None: assert (created / "reports" / "intent-dialogue.md").exists(), created assert (created / "reports" / "intent-confidence.md").exists(), created assert (created / "reports" / "skill-overview.html").exists(), created + assert (created / "reports" / "review-studio.html").exists(), created + assert (created / "reports" / "review-studio.json").exists(), created assert (created / "reports" / "review-viewer.html").exists(), created assert (created / "reports" / "reference-scan.md").exists(), created assert (created / "reports" / "reference-synthesis.md").exists(), created @@ -58,7 +60,10 @@ def main() -> None: init_report_view = init_result["payload"]["report_view"] assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view assert Path(init_report_view["html_report"]).exists(), init_report_view + assert init_report_view["review_studio"].endswith("reports/review-studio.html"), init_report_view + assert Path(init_report_view["review_studio"]).exists(), init_report_view assert "Skill 已创建完成" in init_report_view["message"], init_report_view + assert "Review Studio 2.0" in init_report_view["message"], init_report_view assert "概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view assert "默认使用中文简体" in init_report_view["message"], init_report_view assert "切换英文版" in init_report_view["message"], init_report_view @@ -90,6 +95,7 @@ def main() -> None: assert quickstart_result["ok"], quickstart_result quickstart_root = Path(quickstart_result["payload"]["root"]) assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root + assert (quickstart_root / "reports" / "review-studio.html").exists(), quickstart_root assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root assert (quickstart_root / "reports" / "intent-confidence.md").exists(), quickstart_root assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root @@ -105,9 +111,11 @@ def main() -> None: quickstart_report_view = quickstart_result["payload"]["report_view"] assert quickstart_report_view["html_report"].endswith("reports/skill-overview.html"), quickstart_report_view assert Path(quickstart_report_view["html_report"]).exists(), quickstart_report_view + assert Path(quickstart_report_view["review_studio"]).exists(), quickstart_report_view assert "Skill 已创建完成" in quickstart_report_view["message"], quickstart_report_view assert "默认使用中文简体" in quickstart_report_view["message"], quickstart_report_view assert quickstart_result["payload"]["guidance"]["next_steps"][0].startswith("Open reports/skill-overview.html"), quickstart_result + assert "reports/review-studio.html" in quickstart_result["payload"]["guidance"]["next_steps"][2], quickstart_result assert "audit report" in quickstart_result["payload"]["guidance"]["next_steps"][0], quickstart_result assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["reference_synthesis"].endswith( "reports/reference-synthesis.md" @@ -118,6 +126,9 @@ def main() -> None: assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["system_model"].endswith( "reports/system-model.md" ), quickstart_result + assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["review_studio"].endswith( + "reports/review-studio.html" + ), quickstart_result assert "uncertainty_or_conflict" not in quickstart_result["payload"], quickstart_result quickstart_manifest = json.loads((quickstart_root / "manifest.json").read_text(encoding="utf-8")) assert quickstart_manifest["status"] == "active", quickstart_manifest @@ -162,6 +173,11 @@ def main() -> None: assert review_viewer_result["ok"], review_viewer_result assert review_viewer_result["payload"]["artifacts"]["html"].endswith("reports/review-viewer.html"), review_viewer_result + review_studio_result = run("review-studio", str(created)) + assert review_studio_result["ok"], review_studio_result + assert review_studio_result["payload"]["artifacts"]["html"].endswith("reports/review-studio.html"), review_studio_result + assert review_studio_result["payload"]["summary"]["gate_count"] == 8, review_studio_result + reference_scan_result = run( "reference-scan", str(created),
    路径Path作用Role类型Type
    SKILL.mdSkill 入口文件Skill entrypoint文件file
    README.md人类可读使用说明Human-readable usage guide文件file
    agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
    manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
    references扩展指导与复用资料Extended guidance and reusable notes目录folder
    scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
    evals触发与质量检查Trigger and quality checks目录folder
    reports生成的证据与总结报告Generated evidence and overview artifacts目录folder