Add IR-first packaging and review studio
This commit is contained in:
@@ -2,7 +2,7 @@ PYTHON ?= python3
|
||||
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
|
||||
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
|
||||
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
|
||||
|
||||
eval:
|
||||
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
@@ -58,6 +58,9 @@ skill-atlas-check:
|
||||
review-viewer-check:
|
||||
$(PYTHON) tests/verify_review_viewer.py
|
||||
|
||||
review-studio-check:
|
||||
$(PYTHON) tests/verify_review_studio.py
|
||||
|
||||
feedback-check:
|
||||
$(PYTHON) tests/verify_feedback.py
|
||||
|
||||
@@ -151,11 +154,11 @@ sync-local-install:
|
||||
sync-active-install:
|
||||
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
|
||||
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check review-studio-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
|
||||
ci-test:
|
||||
$(PYTHON) scripts/ci_test.py
|
||||
|
||||
clean:
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_output_eval tests/tmp_conformance tests/tmp_trust tests/tmp_skill_atlas tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_output_eval tests/tmp_conformance tests/tmp_trust tests/tmp_skill_atlas tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
|
||||
find . -type d -name __pycache__ -prune -exec rm -rf {} +
|
||||
|
||||
@@ -22,6 +22,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
- a front-loaded intent dialogue with an intent confidence gate, so the system keeps clarifying when the true job, outputs, exclusions, or standards are still fuzzy
|
||||
- a silent-by-default GitHub benchmark scan plus reference synthesis that studies top public repositories and world-class pattern tracks, then surfaces only real conflicts or uncertainty to the user
|
||||
- a generated visual HTML overview for each newly initialized skill
|
||||
- a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, and release evidence
|
||||
- a side-by-side HTML review studio for first-pass human review
|
||||
- an artifact design profile that defines visual direction, layout patterns, and quality gates for reports, tutorials, dashboards, screenshots, and review pages
|
||||
- a prompt quality profile that abstracts need modeling, RTF mapping, complexity, and quality checks into reviewer-visible evidence instead of bloating `SKILL.md`
|
||||
@@ -30,7 +31,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
- a lightweight feedback log that does not require a full promotion cycle
|
||||
- a baseline compare report for with-skill vs baseline review
|
||||
- a conversation-style, archetype-aware quickstart that steers new packages toward scaffold, production, library, or governed fits
|
||||
- neutral source metadata plus client-specific adapters
|
||||
- Skill IR as the platform-neutral semantic contract plus client-specific adapters
|
||||
- governance, promotion, and portability checks built into the default flow
|
||||
|
||||
## Architecture
|
||||
@@ -84,7 +85,7 @@ Weighted score formula: `sum(score / 10 * weight)`.
|
||||
2. Start with a short, human intent dialogue so the real job, outputs, exclusions, constraints, and standards are explicit.
|
||||
3. Let `quickstart` clarify intent first, then run silent benchmark scan and reference synthesis; it only surfaces explicit questions when intent is still unclear or when there is a real design conflict.
|
||||
4. Use the archetype-aware `quickstart` or the full authoring flow to generate or improve the package in scaffold, production, library, or governed mode.
|
||||
5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then inspect `reports/intent-dialogue.md`, `reports/intent-confidence.md`, `reports/reference-synthesis.md`, `reports/artifact-design-profile.md`, `reports/prompt-quality-profile.md`, `reports/system-model.md`, and `reports/iteration-directions.md` before adding more structure.
|
||||
5. Review the generated `reports/skill-overview.html` first for the bilingual HTML skill report. It defaults to Simplified Chinese and provides an English switch in the top right. Then open `reports/review-studio.html` to inspect release blockers and evidence paths in one page before adding more structure.
|
||||
|
||||
Or use the unified authoring CLI:
|
||||
|
||||
@@ -96,6 +97,7 @@ python3 scripts/yao.py reference-scan my-skill \
|
||||
--user-reference "A product or repo I admire::taste::Learn the clarity and operating standard.::Do not copy wording." \
|
||||
--local-constraint "Current Library Naming::structure::Keep naming aligned with the local skill library.::Do not inherit private references."
|
||||
python3 scripts/yao.py review-viewer my-skill
|
||||
python3 scripts/yao.py review-studio my-skill
|
||||
python3 scripts/yao.py artifact-design-profile my-skill
|
||||
python3 scripts/yao.py prompt-quality-profile my-skill
|
||||
python3 scripts/yao.py system-model my-skill
|
||||
@@ -223,7 +225,7 @@ The homepage panel below is generated from the current eval suite so the family-
|
||||
Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/family_summary.md](reports/family_summary.md)
|
||||
<!-- END:EVAL_RESULTS -->
|
||||
|
||||
- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks
|
||||
- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks and carry IR provenance plus semantic parity metadata
|
||||
- portability score: `100/100` with neutral activation, execution, trust, and degradation metadata preserved across all exported targets
|
||||
- description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately
|
||||
- judge-backed blind eval: root, team frontend review, and governed incident command now pass an independent rubric judge on blind holdout prompts
|
||||
@@ -241,6 +243,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa
|
||||
- promotion decisions are published in [reports/promotion_decisions.md](reports/promotion_decisions.md)
|
||||
- candidate lifecycle states are published in [reports/candidate_registry.md](reports/candidate_registry.md)
|
||||
- lightweight with-skill vs baseline comparison is published in [reports/baseline-compare.md](reports/baseline-compare.md)
|
||||
- Review Studio 2.0 gate evidence is published in [reports/review-studio.html](reports/review-studio.html)
|
||||
- context budget summaries are tracked in [reports/context_budget.md](reports/context_budget.md)
|
||||
- portability status is tracked in [reports/portability_score.md](reports/portability_score.md)
|
||||
|
||||
@@ -359,7 +362,7 @@ Utility scripts that make the meta-skill operational:
|
||||
- `governance_check.py`: validates owner, review cadence, lifecycle stage, and maturity metadata
|
||||
- `render_context_reports.py`: generates root and example context-budget reports plus a shared context summary
|
||||
- `render_regression_history.py`: turns milestone snapshots into a readable regression history report
|
||||
- `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation
|
||||
- `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation
|
||||
- `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage
|
||||
- `render_skill_overview.py`: generates the white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, top-right language switch, v2 scorecard, inline SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap
|
||||
- `export_skill_ir.py`: exports the 2.0 platform-neutral Skill IR contract from `SKILL.md`, manifest, interface metadata, evals, resources, and reports
|
||||
@@ -434,7 +437,7 @@ The typical flow is:
|
||||
3. choose scaffold, production, or library mode
|
||||
4. generate the package
|
||||
5. run the sizing and trigger checks if needed
|
||||
6. export target-specific compatibility artifacts
|
||||
6. export target-specific compatibility artifacts from the Skill IR contract
|
||||
|
||||
### 3. Export compatibility artifacts
|
||||
|
||||
|
||||
@@ -32,17 +32,19 @@ Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes](
|
||||
6. Add only folders and gates that earn their keep.
|
||||
7. Surface the top three next iteration directions.
|
||||
|
||||
Core playbooks: [Method](references/skill-engineering-method.md), [Intent Dialogue](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Prompt Engineering](references/prompt-engineering-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md).
|
||||
Core playbooks: [Method](references/skill-engineering-method.md), [Intent](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Review Studio](references/review-studio-method.md).
|
||||
|
||||
## Skill OS 2.0 Gates
|
||||
|
||||
For production, library, governed, or team-distributed work, use Skill IR, trigger + output eval, Skill Atlas, runtime conformance, trust report, and Review Studio before release.
|
||||
|
||||
## First-Turn Style
|
||||
|
||||
When the skill first activates:
|
||||
|
||||
- open warmly, like a thoughtful teacher or design partner
|
||||
- start from the user's work and desired outcome before asking for structure
|
||||
- start from the user's work and desired outcome before structure
|
||||
- ask only `2-3` key questions unless the user already gave enough detail
|
||||
- let the user answer naturally first; offer a tiny scaffold only as a shortcut
|
||||
- avoid cold field lists; turn benchmark work into one recommendation unless uncertainty or conflict needs a visible call
|
||||
- avoid cold field lists; surface benchmark choices only when uncertainty or conflict needs a call
|
||||
|
||||
Chinese conversations should sound soft and companion-like rather than procedural.
|
||||
|
||||
@@ -50,14 +52,8 @@ Opening patterns: [Intent Dialogue](references/intent-dialogue.md).
|
||||
|
||||
## Output Contract
|
||||
|
||||
Unless the user asks otherwise, produce:
|
||||
|
||||
1. a working skill directory
|
||||
2. a `SKILL.md`
|
||||
3. aligned `agents/interface.yaml`
|
||||
4. optional `references/`, `scripts/`, `evals/`, `reports/`, and `manifest.json` when justified
|
||||
5. a short summary of boundary, exclusions, gates, and next steps
|
||||
Unless the user asks otherwise, produce a working skill directory with `SKILL.md`, aligned `agents/interface.yaml`, justified assets, and a short summary of boundary, exclusions, gates, and next steps.
|
||||
|
||||
## Reference Map
|
||||
|
||||
Primary references: [Method](references/skill-engineering-method.md), [Reference Scan](references/reference-scan.md), [Artifact Design](references/artifact-design-doctrine.md), [Prompt Engineering](references/prompt-engineering-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md).
|
||||
Primary references: [Method](references/skill-engineering-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md), [Governance](references/governance.md).
|
||||
|
||||
+3
-3
@@ -15,8 +15,8 @@ Contents:
|
||||
- `improved_description.txt`: current stronger trigger description
|
||||
- `../reports/description_optimization*.{json,md}`: generated route-optimization reports for the root skill
|
||||
- `failure-cases.md`: current weak spots and regression targets
|
||||
- `packaging_expectations.json`: required packaging behaviors for supported targets
|
||||
- `output/`: Output Eval Lab cases and schema for with-skill vs baseline assertion grading
|
||||
- `packaging_expectations.json`: required packaging behaviors for supported targets, including IR provenance and semantic parity fields
|
||||
- `output/`: Output Eval Lab cases, fixtures, and schema for with-skill vs baseline assertion grading
|
||||
- `../reports/`: generated suite JSON plus the homepage-visible family summary panel source
|
||||
|
||||
Use:
|
||||
@@ -57,4 +57,4 @@ Regression scope now includes:
|
||||
- judge-backed blind-holdout verification that adds a rubric-based second opinion for blind prompts
|
||||
- calibration summaries that surface score gaps, threshold margins, and risk bands for each acceptance gate
|
||||
- family-level drift history that records which blind and adversarial families stay clean over time
|
||||
- output-quality scorecards that compare without-skill and with-skill artifacts through assertion grading
|
||||
- output-quality scorecards that compare without-skill and with-skill artifacts through assertion grading, including near-neighbor and file-backed governed cases
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
{"id":"skill-package-contract","prompt":"Turn this repeated workflow into a reusable team skill package.","baseline_output":"I can write a prompt for that workflow and include a short checklist.","with_skill_output":"Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/.","assertions":[{"id":"has-entrypoint","description":"Output names the routeable Skill entrypoint.","required":["SKILL.md"],"weight":1,"failure_type":"missing_entrypoint"},{"id":"has-interface","description":"Output includes neutral interface metadata.","required":["agents/interface.yaml"],"weight":1,"failure_type":"missing_interface"},{"id":"has-report","description":"Output points reviewer to generated overview report.","required":["reports/skill-overview.html"],"weight":1,"failure_type":"missing_review_artifact"},{"id":"has-resource-boundary","description":"Output preserves entrypoint/resource separation.","required":["references/","scripts/","reports/"],"weight":1,"failure_type":"resource_boundary"}],"human_review":{"expected_winner":"with_skill"}}
|
||||
{"id":"output-eval-expectation","prompt":"Upgrade this production skill so we know whether the generated output is better than baseline.","baseline_output":"Add more examples and run the trigger eval again.","with_skill_output":"Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md.","assertions":[{"id":"has-baseline-comparison","description":"Output explicitly compares with-skill and baseline outputs.","required":["with-skill","baseline"],"weight":1,"failure_type":"missing_baseline_delta"},{"id":"has-assertion-grading","description":"Output includes objective assertion grading.","required":["assertion grading"],"weight":1,"failure_type":"missing_assertions"},{"id":"has-scorecard","description":"Output produces a scorecard report path.","required":["reports/output_quality_scorecard.md"],"weight":1,"failure_type":"missing_scorecard"},{"id":"has-failure-taxonomy","description":"Output records failure taxonomy.","required":["failure taxonomy"],"weight":1,"failure_type":"missing_failure_taxonomy"}],"human_review":{"expected_winner":"with_skill"}}
|
||||
{"id":"ir-before-packaging","prompt":"Package this skill for OpenAI, Claude, Agent Skills, and generic targets.","baseline_output":"Export folders for each platform and adjust files as needed.","with_skill_output":"Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract.","assertions":[{"id":"has-ir-first","description":"Output requires Skill IR before packaging.","required":["Skill IR first"],"weight":1,"failure_type":"missing_ir"},{"id":"has-core-ir-fields","description":"Output lists core IR contract fields.","required":["job_to_be_done","trigger_surface","eval_plan","governance"],"weight":1,"failure_type":"incomplete_ir_contract"},{"id":"has-targets","description":"Output names requested runtime targets.","required":["OpenAI","Claude","Agent Skills","generic"],"weight":1,"failure_type":"missing_targets"},{"id":"has-semantic-preservation","description":"Output says targets must preserve the capability contract.","required":["preserve the same capability contract"],"weight":1,"failure_type":"semantic_drift"}],"human_review":{"expected_winner":"with_skill"}}
|
||||
{"id":"near-neighbor-boundary","prompt":"I only need a one-off summary of these notes, not a reusable process.","baseline_output":"Create a SKILL.md and a reusable workflow anyway so future notes can use it.","with_skill_output":"Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract.","assertions":[{"id":"declines-skill","description":"Output refuses unnecessary skill creation for one-off work.","required":["Do not create a skill"],"weight":1,"failure_type":"over_trigger"},{"id":"names-near-neighbor","description":"Output labels the request as near-neighbor instead of owned work.","required":["near-neighbor"],"weight":1,"failure_type":"missing_boundary_language"},{"id":"requires-reuse-signal","description":"Output asks for repeat-use evidence before packaging.","required":["repeated use","reusable output contract"],"weight":1,"failure_type":"missing_reuse_gate"}],"human_review":{"expected_winner":"with_skill"},"metadata":{"case_type":"near_neighbor","tier":"production"}}
|
||||
{"id":"file-backed-governed-package","prompt":"Turn the attached release brief source into a governed skill package.","input_files":["fixtures/release-brief-source.md"],"baseline_output":"Draft a release announcement with the changelog, support notes, and owner name.","with_skill_output":"Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them.","assertions":[{"id":"uses-file-backed-evidence","description":"Output names file-backed source evidence.","required":["file-backed fixture","input_files"],"weight":1,"failure_type":"missing_file_fixture"},{"id":"has-governance","description":"Output preserves governed ownership metadata.","required":["owner","review cadence"],"weight":1,"failure_type":"missing_governance"},{"id":"has-output-contract","description":"Output preserves output and rollback boundaries.","required":["output contract","rollback boundary"],"weight":1,"failure_type":"missing_contract_boundary"},{"id":"has-trust-and-scorecard","description":"Output requires trust and output scorecard artifacts.","required":["trust report","reports/output_quality_scorecard.md"],"weight":1,"failure_type":"missing_release_evidence"},{"id":"does-not-invent-evidence","description":"Output forbids invented launch evidence.","required":["missing evidence"],"forbidden":["invented launch metrics"],"weight":1,"failure_type":"invented_evidence"}],"human_review":{"expected_winner":"with_skill"},"metadata":{"case_type":"boundary","tier":"governed"}}
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# Release Brief Source Fixture
|
||||
|
||||
## Source
|
||||
|
||||
- Raw changelog notes
|
||||
- Support escalation summary
|
||||
- Rollback checklist
|
||||
- Internal launch owner
|
||||
|
||||
## Required Output Constraints
|
||||
|
||||
- Produce a reusable skill package, not a one-off announcement.
|
||||
- Preserve owner, review cadence, input files, output contract, and rollback boundary.
|
||||
- Surface missing evidence instead of inventing launch metrics.
|
||||
@@ -7,6 +7,11 @@
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
|
||||
@@ -29,12 +29,20 @@ When `--expectations` is provided:
|
||||
|
||||
## Source Of Truth
|
||||
|
||||
The neutral source remains:
|
||||
The platform-neutral semantic source is Skill IR when it exists:
|
||||
|
||||
- `reports/skill-ir.json`
|
||||
- `skill-ir/examples/<skill-name>.json`
|
||||
|
||||
The structural validation sources remain:
|
||||
|
||||
- `SKILL.md`
|
||||
- `agents/interface.yaml`
|
||||
|
||||
Target-specific metadata is generated at packaging time.
|
||||
Target-specific metadata is generated at packaging time. The adapter must carry
|
||||
`ir_source`, `ir_schema_version`, `job_to_be_done`, `semantic_contract`, and
|
||||
`semantic_parity` so reviewers can see whether the target preserved the core
|
||||
skill meaning or fell back to frontmatter-only metadata.
|
||||
|
||||
## Portability Model
|
||||
|
||||
@@ -44,5 +52,6 @@ The packaging layer now preserves four portable semantics from the neutral sourc
|
||||
- execution
|
||||
- trust
|
||||
- degradation
|
||||
- platform-neutral skill meaning from Skill IR
|
||||
|
||||
This means portability is not just "can it export a file?" but also "does the exported target preserve the source package's activation and safety assumptions?"
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# Review Studio 2.0 Method
|
||||
|
||||
Review Studio is the release-facing audit surface for a skill package. It does not replace the detailed reports; it turns them into one reviewer decision page.
|
||||
|
||||
## Purpose
|
||||
|
||||
- Show release blockers and warnings before the package deepens.
|
||||
- Link every gate back to a concrete evidence artifact.
|
||||
- Keep review flow vertical: summary first, gates second, supporting details after.
|
||||
- Avoid hiding output quality, runtime, trust, and portfolio issues across separate pages.
|
||||
|
||||
## Required Gates
|
||||
|
||||
1. Intent Canvas: intent confidence and unresolved input/output/exclusion gaps.
|
||||
2. Trigger Lab: route scorecard, misroutes, ambiguous cases, and near-neighbor safety.
|
||||
3. Output Lab: with-skill vs baseline delta, case count, file-backed cases, near-neighbor cases, and boundary cases.
|
||||
4. Context Budget: initial load, budget tier, warnings, and quality density.
|
||||
5. Runtime Matrix: target conformance pass/fail and degradation notes.
|
||||
6. Trust Report: secret scan, script surface, dependency pinning, network/interactive flags, and package hash.
|
||||
7. Skill Atlas: route collisions, stale skills, owner gaps, and no-route opportunities.
|
||||
8. Release Notes: promotion status, migration notes, known gaps, and next move.
|
||||
|
||||
## Gate Semantics
|
||||
|
||||
- `pass`: evidence is present and the gate is satisfied.
|
||||
- `warn`: review can continue, but the issue must be visible before release.
|
||||
- `block`: do not claim production, library, governed, or public readiness until fixed.
|
||||
|
||||
For library and governed skills, Output Lab should have at least five cases and cover file-backed, near-neighbor, and boundary scenarios.
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"skill_name": "yao-meta-skill",
|
||||
"design_system": "evidence frame",
|
||||
"design_system": "review studio",
|
||||
"primary_artifact": {
|
||||
"key": "code_or_cli",
|
||||
"label": "Code, CLI, or implementation guide",
|
||||
@@ -33,6 +33,15 @@
|
||||
],
|
||||
"direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail."
|
||||
},
|
||||
{
|
||||
"key": "review_viewer",
|
||||
"label": "Review viewer",
|
||||
"score": 1,
|
||||
"matched_keywords": [
|
||||
"review"
|
||||
],
|
||||
"direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix."
|
||||
},
|
||||
{
|
||||
"key": "visual_capture",
|
||||
"label": "Screenshot or visual evidence",
|
||||
@@ -82,8 +91,8 @@
|
||||
"Keep the first screen useful without requiring the reader to parse every detail.",
|
||||
"Use tables only for comparisons; move explanations below the table.",
|
||||
"Keep source notes readable without flooding the body with markers.",
|
||||
"Never invent missing screenshots or visual states.",
|
||||
"Record source, viewport, and crop intent."
|
||||
"Make differences visible instead of hiding them in prose.",
|
||||
"Separate author-facing recommendations from reviewer-only evidence."
|
||||
],
|
||||
"anti_patterns": [
|
||||
"Do not copy Kami's fixed parchment background as a default.",
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Artifact Design Profile
|
||||
|
||||
Skill: `yao-meta-skill`
|
||||
Design system: `evidence frame`
|
||||
Design system: `review studio`
|
||||
|
||||
## Primary Artifact Direction
|
||||
|
||||
@@ -21,6 +21,11 @@ Execution-focused technical artifact with environment assumptions, copyable comm
|
||||
- Score: `2`
|
||||
- Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
|
||||
|
||||
### Review viewer
|
||||
- Matched keywords: review
|
||||
- Score: `1`
|
||||
- Direction: Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix.
|
||||
|
||||
### Screenshot or visual evidence
|
||||
- Matched keywords: screenshot
|
||||
- Score: `1`
|
||||
@@ -67,8 +72,8 @@ Execution-focused technical artifact with environment assumptions, copyable comm
|
||||
- Keep the first screen useful without requiring the reader to parse every detail.
|
||||
- Use tables only for comparisons; move explanations below the table.
|
||||
- Keep source notes readable without flooding the body with markers.
|
||||
- Never invent missing screenshots or visual states.
|
||||
- Record source, viewport, and crop intent.
|
||||
- Make differences visible instead of hiding them in prose.
|
||||
- Separate author-facing recommendations from reviewer-only evidence.
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
|
||||
@@ -5,13 +5,13 @@
|
||||
"stats": {
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 810,
|
||||
"other_text_tokens": 548453,
|
||||
"estimated_initial_load_tokens": 986,
|
||||
"estimated_total_text_tokens": 549263,
|
||||
"relevant_file_count": 251,
|
||||
"skill_body_tokens": 734,
|
||||
"other_text_tokens": 606468,
|
||||
"estimated_initial_load_tokens": 910,
|
||||
"estimated_total_text_tokens": 607202,
|
||||
"relevant_file_count": 241,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
"quality_density": 131.8
|
||||
"quality_density": 142.9
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
| Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- |
|
||||
| root | `.` | `production` | 1000 | 986 | 810 | 131.8 | - | ok |
|
||||
| root | `.` | `production` | 1000 | 910 | 734 | 142.9 | - | ok |
|
||||
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok |
|
||||
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok |
|
||||
|
||||
|
||||
@@ -6,9 +6,9 @@
|
||||
"path": ".",
|
||||
"budget_tier": "production",
|
||||
"budget_limit": 1000,
|
||||
"initial_tokens": 986,
|
||||
"skill_body_tokens": 810,
|
||||
"quality_density": 131.8,
|
||||
"initial_tokens": 910,
|
||||
"skill_body_tokens": 734,
|
||||
"quality_density": 142.9,
|
||||
"unused_resource_dirs": [],
|
||||
"ok": true,
|
||||
"warnings": []
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
"ok": true,
|
||||
"cases": "evals/output/cases.jsonl",
|
||||
"summary": {
|
||||
"case_count": 3,
|
||||
"case_count": 5,
|
||||
"file_backed_case_count": 1,
|
||||
"near_neighbor_case_count": 1,
|
||||
"boundary_case_count": 1,
|
||||
"baseline_pass_rate": 0.0,
|
||||
"with_skill_pass_rate": 100.0,
|
||||
"delta": 100.0,
|
||||
@@ -14,6 +17,8 @@
|
||||
{
|
||||
"id": "skill-package-contract",
|
||||
"prompt": "Turn this repeated workflow into a reusable team skill package.",
|
||||
"input_files": [],
|
||||
"metadata": {},
|
||||
"baseline": {
|
||||
"score": 0.0,
|
||||
"passed_count": 0,
|
||||
@@ -166,6 +171,8 @@
|
||||
{
|
||||
"id": "output-eval-expectation",
|
||||
"prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.",
|
||||
"input_files": [],
|
||||
"metadata": {},
|
||||
"baseline": {
|
||||
"score": 0.0,
|
||||
"passed_count": 0,
|
||||
@@ -316,6 +323,8 @@
|
||||
{
|
||||
"id": "ir-before-packaging",
|
||||
"prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.",
|
||||
"input_files": [],
|
||||
"metadata": {},
|
||||
"baseline": {
|
||||
"score": 0.0,
|
||||
"passed_count": 0,
|
||||
@@ -472,6 +481,322 @@
|
||||
"delta": 100.0,
|
||||
"winner": "with_skill",
|
||||
"failure_taxonomy": []
|
||||
},
|
||||
{
|
||||
"id": "near-neighbor-boundary",
|
||||
"prompt": "I only need a one-off summary of these notes, not a reusable process.",
|
||||
"input_files": [],
|
||||
"metadata": {
|
||||
"case_type": "near_neighbor",
|
||||
"tier": "production"
|
||||
},
|
||||
"baseline": {
|
||||
"score": 0.0,
|
||||
"passed_count": 0,
|
||||
"failed_count": 3,
|
||||
"checks": [
|
||||
{
|
||||
"id": "declines-skill",
|
||||
"description": "Output refuses unnecessary skill creation for one-off work.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "over_trigger",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"Do not create a skill"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "names-near-neighbor",
|
||||
"description": "Output labels the request as near-neighbor instead of owned work.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_boundary_language",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"near-neighbor"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "requires-reuse-signal",
|
||||
"description": "Output asks for repeat-use evidence before packaging.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_reuse_gate",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"repeated use",
|
||||
"reusable output contract"
|
||||
],
|
||||
"present_forbidden": []
|
||||
}
|
||||
],
|
||||
"failed": [
|
||||
{
|
||||
"id": "declines-skill",
|
||||
"description": "Output refuses unnecessary skill creation for one-off work.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "over_trigger",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"Do not create a skill"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "names-near-neighbor",
|
||||
"description": "Output labels the request as near-neighbor instead of owned work.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_boundary_language",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"near-neighbor"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "requires-reuse-signal",
|
||||
"description": "Output asks for repeat-use evidence before packaging.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_reuse_gate",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"repeated use",
|
||||
"reusable output contract"
|
||||
],
|
||||
"present_forbidden": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"with_skill": {
|
||||
"score": 100.0,
|
||||
"passed_count": 3,
|
||||
"failed_count": 0,
|
||||
"checks": [
|
||||
{
|
||||
"id": "declines-skill",
|
||||
"description": "Output refuses unnecessary skill creation for one-off work.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "over_trigger",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "names-near-neighbor",
|
||||
"description": "Output labels the request as near-neighbor instead of owned work.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_boundary_language",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "requires-reuse-signal",
|
||||
"description": "Output asks for repeat-use evidence before packaging.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_reuse_gate",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
}
|
||||
],
|
||||
"failed": []
|
||||
},
|
||||
"delta": 100.0,
|
||||
"winner": "with_skill",
|
||||
"failure_taxonomy": []
|
||||
},
|
||||
{
|
||||
"id": "file-backed-governed-package",
|
||||
"prompt": "Turn the attached release brief source into a governed skill package.",
|
||||
"input_files": [
|
||||
"fixtures/release-brief-source.md"
|
||||
],
|
||||
"metadata": {
|
||||
"case_type": "boundary",
|
||||
"tier": "governed"
|
||||
},
|
||||
"baseline": {
|
||||
"score": 0.0,
|
||||
"passed_count": 0,
|
||||
"failed_count": 5,
|
||||
"checks": [
|
||||
{
|
||||
"id": "uses-file-backed-evidence",
|
||||
"description": "Output names file-backed source evidence.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_file_fixture",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"file-backed fixture",
|
||||
"input_files"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-governance",
|
||||
"description": "Output preserves governed ownership metadata.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_governance",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"review cadence"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-output-contract",
|
||||
"description": "Output preserves output and rollback boundaries.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_contract_boundary",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"output contract",
|
||||
"rollback boundary"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-trust-and-scorecard",
|
||||
"description": "Output requires trust and output scorecard artifacts.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_release_evidence",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"trust report",
|
||||
"reports/output_quality_scorecard.md"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "does-not-invent-evidence",
|
||||
"description": "Output forbids invented launch evidence.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "invented_evidence",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"missing evidence"
|
||||
],
|
||||
"present_forbidden": []
|
||||
}
|
||||
],
|
||||
"failed": [
|
||||
{
|
||||
"id": "uses-file-backed-evidence",
|
||||
"description": "Output names file-backed source evidence.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_file_fixture",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"file-backed fixture",
|
||||
"input_files"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-governance",
|
||||
"description": "Output preserves governed ownership metadata.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_governance",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"review cadence"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-output-contract",
|
||||
"description": "Output preserves output and rollback boundaries.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_contract_boundary",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"output contract",
|
||||
"rollback boundary"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-trust-and-scorecard",
|
||||
"description": "Output requires trust and output scorecard artifacts.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_release_evidence",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"trust report",
|
||||
"reports/output_quality_scorecard.md"
|
||||
],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "does-not-invent-evidence",
|
||||
"description": "Output forbids invented launch evidence.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "invented_evidence",
|
||||
"passed": false,
|
||||
"missing": [
|
||||
"missing evidence"
|
||||
],
|
||||
"present_forbidden": []
|
||||
}
|
||||
]
|
||||
},
|
||||
"with_skill": {
|
||||
"score": 100.0,
|
||||
"passed_count": 5,
|
||||
"failed_count": 0,
|
||||
"checks": [
|
||||
{
|
||||
"id": "uses-file-backed-evidence",
|
||||
"description": "Output names file-backed source evidence.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_file_fixture",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-governance",
|
||||
"description": "Output preserves governed ownership metadata.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_governance",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-output-contract",
|
||||
"description": "Output preserves output and rollback boundaries.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_contract_boundary",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "has-trust-and-scorecard",
|
||||
"description": "Output requires trust and output scorecard artifacts.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "missing_release_evidence",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
},
|
||||
{
|
||||
"id": "does-not-invent-evidence",
|
||||
"description": "Output forbids invented launch evidence.",
|
||||
"weight": 1.0,
|
||||
"failure_type": "invented_evidence",
|
||||
"passed": true,
|
||||
"missing": [],
|
||||
"present_forbidden": []
|
||||
}
|
||||
],
|
||||
"failed": []
|
||||
},
|
||||
"delta": 100.0,
|
||||
"winner": "with_skill",
|
||||
"failure_taxonomy": []
|
||||
}
|
||||
],
|
||||
"failures": [],
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
This v0 scorecard compares static without-skill and with-skill outputs using assertion grading.
|
||||
|
||||
- Cases: `3`
|
||||
- Cases: `5`
|
||||
- Baseline pass rate: `0.0`
|
||||
- With-skill pass rate: `100.0`
|
||||
- Delta: `100.0`
|
||||
@@ -16,6 +16,8 @@ This v0 scorecard compares static without-skill and with-skill outputs using ass
|
||||
| skill-package-contract | 0.0 | 100.0 | 100.0 | with_skill | None |
|
||||
| output-eval-expectation | 0.0 | 100.0 | 100.0 | with_skill | None |
|
||||
| ir-before-packaging | 0.0 | 100.0 | 100.0 | with_skill | None |
|
||||
| near-neighbor-boundary | 0.0 | 100.0 | 100.0 | with_skill | None |
|
||||
| file-backed-governed-package | 0.0 | 100.0 | 100.0 | with_skill | None |
|
||||
|
||||
## Failure Taxonomy
|
||||
|
||||
|
||||
@@ -10,10 +10,12 @@
|
||||
"trust_level": "local",
|
||||
"remote_inline_execution": "forbid",
|
||||
"degradation_coverage": 3,
|
||||
"snapshot_count": 3
|
||||
"snapshot_count": 3,
|
||||
"ir_source": "skill-ir/examples/yao-meta-skill.json",
|
||||
"ir_schema_version": "2.0.0"
|
||||
},
|
||||
"breakdown": {
|
||||
"canonical_neutral_source": 15,
|
||||
"platform_neutral_semantic_source": 15,
|
||||
"adapter_target_coverage": 15,
|
||||
"activation_metadata": 10,
|
||||
"execution_metadata": 10,
|
||||
|
||||
@@ -8,10 +8,12 @@
|
||||
- shell: `bash`
|
||||
- trust level: `local`
|
||||
- remote inline execution: `forbid`
|
||||
- Skill IR source: `skill-ir/examples/yao-meta-skill.json`
|
||||
- Skill IR schema: `2.0.0`
|
||||
|
||||
| Component | Score |
|
||||
| --- | ---: |
|
||||
| `canonical_neutral_source` | 15 |
|
||||
| `platform_neutral_semantic_source` | 15 |
|
||||
| `adapter_target_coverage` | 15 |
|
||||
| `activation_metadata` | 10 |
|
||||
| `execution_metadata` | 10 |
|
||||
|
||||
@@ -33,17 +33,6 @@
|
||||
"task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.",
|
||||
"format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant."
|
||||
},
|
||||
{
|
||||
"key": "teaching_guidance",
|
||||
"label": "Teaching guidance",
|
||||
"score": 1,
|
||||
"matched_keywords": [
|
||||
"teach"
|
||||
],
|
||||
"role_guidance": "Use a teacher role that adapts to learner level and avoids overloading the first pass.",
|
||||
"task_guidance": "Explain through progressive steps, examples, and visible success checks.",
|
||||
"format_guidance": "Return learner-facing sections, worked examples, checkpoints, and common mistakes."
|
||||
},
|
||||
{
|
||||
"key": "prompt_engineering",
|
||||
"label": "Prompt engineering",
|
||||
@@ -58,7 +47,7 @@
|
||||
],
|
||||
"complexity": {
|
||||
"band": "complex",
|
||||
"score": 6,
|
||||
"score": 5,
|
||||
"reason": "multiple inputs, constraints, or task families require tradeoff handling"
|
||||
},
|
||||
"need_model": {
|
||||
|
||||
@@ -12,7 +12,7 @@ Overall quality score: `82.0/100`
|
||||
## Complexity
|
||||
|
||||
- Band: `complex`
|
||||
- Score: `6`
|
||||
- Score: `5`
|
||||
- Reason: multiple inputs, constraints, or task families require tradeoff handling
|
||||
|
||||
## Need Model
|
||||
@@ -67,13 +67,6 @@ Overall quality score: `82.0/100`
|
||||
- Task: Convert the job into ordered steps with validation checks and stop conditions.
|
||||
- Format: Return a runbook-like handoff with commands, checks, owners, and next actions when relevant.
|
||||
|
||||
### Teaching guidance
|
||||
- Score: `1`
|
||||
- Keywords: teach
|
||||
- Role: Use a teacher role that adapts to learner level and avoids overloading the first pass.
|
||||
- Task: Explain through progressive steps, examples, and visible success checks.
|
||||
- Format: Return learner-facing sections, worked examples, checkpoints, and common mistakes.
|
||||
|
||||
### Prompt engineering
|
||||
- Score: `1`
|
||||
- Keywords: prompt
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Yao Meta Skill Review Studio 2.0</title>
|
||||
<style>
|
||||
:root {
|
||||
--ink: #1B365D;
|
||||
--text: #24201d;
|
||||
--muted: #746d66;
|
||||
--line: #e7ded2;
|
||||
--soft: #faf8f5;
|
||||
--pass: #1e6b52;
|
||||
--warn: #9a6718;
|
||||
--block: #9b2c2c;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
html { scroll-behavior: smooth; }
|
||||
body {
|
||||
margin: 0;
|
||||
background: #ffffff;
|
||||
color: var(--text);
|
||||
font-family: Georgia, "Times New Roman", "Songti SC", serif;
|
||||
line-height: 1.58;
|
||||
}
|
||||
nav {
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
padding: 10px 16px;
|
||||
background: rgba(255,255,255,0.94);
|
||||
border-bottom: 1px solid var(--line);
|
||||
backdrop-filter: blur(10px);
|
||||
}
|
||||
nav a {
|
||||
color: var(--ink);
|
||||
text-decoration: none;
|
||||
font-size: 14px;
|
||||
padding: 7px 10px;
|
||||
border-radius: 6px;
|
||||
}
|
||||
nav a:hover { background: var(--soft); }
|
||||
main { max-width: 1180px; margin: 0 auto; padding: 44px 28px 76px; }
|
||||
header { border-bottom: 1px solid var(--line); padding-bottom: 28px; margin-bottom: 28px; }
|
||||
.eyebrow { color: var(--ink); font-size: 14px; letter-spacing: .08em; text-transform: uppercase; }
|
||||
h1, h2, h3 { color: var(--text); font-weight: 500; margin: 0; letter-spacing: 0; }
|
||||
h1 { font-size: clamp(34px, 5vw, 64px); line-height: 1.03; max-width: 920px; margin-top: 12px; }
|
||||
h2 { font-size: 30px; margin-bottom: 14px; }
|
||||
h3 { font-size: 19px; }
|
||||
p { margin: 0; }
|
||||
.lede { max-width: 820px; color: var(--muted); font-size: 20px; margin-top: 18px; }
|
||||
.decision {
|
||||
display: inline-flex;
|
||||
align-items: baseline;
|
||||
gap: 12px;
|
||||
margin-top: 24px;
|
||||
padding: 12px 16px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
color: var(--ink);
|
||||
background: var(--soft);
|
||||
}
|
||||
.decision strong { font-size: 28px; }
|
||||
section { padding: 30px 0; border-bottom: 1px solid var(--line); scroll-margin-top: 76px; }
|
||||
.metrics, .gates {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
}
|
||||
.gates { grid-template-columns: repeat(4, minmax(0, 1fr)); }
|
||||
.metric, .gate {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
background: #fff;
|
||||
min-width: 0;
|
||||
}
|
||||
.metric span, .gate span { display: block; color: var(--muted); font-size: 13px; }
|
||||
.metric strong { display: block; color: var(--ink); font-size: 34px; line-height: 1.1; margin: 8px 0; }
|
||||
.metric p, .gate p, .gate footer, .issues span, .evidence span { color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }
|
||||
.gate { display: flex; flex-direction: column; gap: 10px; }
|
||||
.gate.pass { border-top: 4px solid var(--pass); }
|
||||
.gate.warn { border-top: 4px solid var(--warn); }
|
||||
.gate.block { border-top: 4px solid var(--block); }
|
||||
.gate footer { border-top: 1px solid var(--line); padding-top: 10px; }
|
||||
a { color: var(--ink); }
|
||||
.twocol {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
|
||||
gap: 22px;
|
||||
align-items: start;
|
||||
}
|
||||
.panel {
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 18px;
|
||||
background: #fff;
|
||||
}
|
||||
.panel p { color: var(--muted); }
|
||||
.issues, .evidence {
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
.issues li, .evidence li {
|
||||
border-left: 3px solid var(--line);
|
||||
padding-left: 12px;
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
}
|
||||
.muted { color: var(--muted); }
|
||||
@media (max-width: 980px) {
|
||||
.metrics, .gates, .twocol { grid-template-columns: 1fr; }
|
||||
main { padding: 32px 18px 60px; }
|
||||
nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; }
|
||||
nav a { flex: 0 0 auto; }
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav><a href='#overview'>审查总览</a><a href='#intent'>意图画布</a><a href='#trigger'>触发实验</a><a href='#output'>输出实验</a><a href='#runtime'>运行矩阵</a><a href='#trust'>信任报告</a><a href='#atlas'>组合治理</a><a href='#release'>发布路线</a></nav>
|
||||
<main>
|
||||
<header id="overview">
|
||||
<div class="eyebrow">Review Studio 2.0</div>
|
||||
<h1>Yao Meta Skill</h1>
|
||||
<p class="lede">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</p>
|
||||
<div class="decision">
|
||||
<span>审查结论</span>
|
||||
<strong>review</strong>
|
||||
<span>Score 87/100</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section>
|
||||
<h2>核心指标</h2>
|
||||
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>4 targets in platform-neutral contract</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>50 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article></div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>审查闸门</h2>
|
||||
<div class="gates"><article class='gate warn'><div><span>关注</span><h3>意图画布</h3></div><p>intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 910/1000; quality density 142.9</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>信任报告</h3></div><p>0 secrets; 50 scripts; 2 network-capable scripts</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>组合治理</h3></div><p>12 skills; 5 route collisions; 9 owner gaps; 10 stale</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked</p><footer>reports/promotion_decisions.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
|
||||
</section>
|
||||
|
||||
<div class="twocol">
|
||||
<section><h2>阻断事项</h2><p class='muted'>无。</p></section>
|
||||
<section><h2>关注事项</h2><ul class='issues'><li><strong>意图画布</strong><span>intent confidence 30/100; Pause before deep authoring and close the highest-leverage gaps first.</span></li><li><strong>信任报告</strong><span>0 secrets; 50 scripts; 2 network-capable scripts</span></li><li><strong>组合治理</strong><span>12 skills; 5 route collisions; 9 owner gaps; 10 stale</span></li></ul></section>
|
||||
</div>
|
||||
|
||||
<section id="intent" class="twocol">
|
||||
<div class="panel">
|
||||
<h2>意图画布</h2>
|
||||
<p>Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</p>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<h2>证据路径</h2>
|
||||
<ul class="evidence"><li><strong>skill_overview</strong><span>reports/skill-overview.html</span></li><li><strong>review_viewer</strong><span>reports/review-viewer.html</span></li><li><strong>output_eval</strong><span>reports/output_quality_scorecard.md</span></li><li><strong>runtime_conformance</strong><span>reports/conformance_matrix.md</span></li><li><strong>trust_report</strong><span>reports/security_trust_report.md</span></li><li><strong>skill_atlas</strong><span>reports/skill_atlas.html</span></li><li><strong>migration</strong><span>docs/migration-v2.md</span></li></ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="trigger" class="twocol">
|
||||
<div class="panel"><h2>触发实验</h2><p>13 trigger cases; 0 misroutes; 0 ambiguous</p></div>
|
||||
<div class="panel"><h2>组合治理</h2><p>{'skill_count': 12, 'route_collision_count': 5, 'owner_gap_count': 9, 'stale_count': 10, 'shared_resource_count': 0, 'no_route_opportunity_count': 0}</p></div>
|
||||
</section>
|
||||
|
||||
<section id="output" class="twocol">
|
||||
<div class="panel"><h2>输出实验</h2><p>{'case_count': 5, 'file_backed_case_count': 1, 'near_neighbor_case_count': 1, 'boundary_case_count': 1, 'baseline_pass_rate': 0.0, 'with_skill_pass_rate': 100.0, 'delta': 100.0, 'regression_count': 0, 'gate_pass': True, 'failure_taxonomy': []}</p></div>
|
||||
<div class="panel"><h2>发布标准</h2><p>Governed 和 Library 至少需要 5 个 output eval cases,并覆盖 file-backed、near-neighbor 和 boundary case。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="runtime" class="twocol">
|
||||
<div class="panel"><h2>运行矩阵</h2><p>{'target_count': 5, 'pass_count': 5, 'fail_count': 0}</p></div>
|
||||
<div class="panel"><h2>上下文</h2><p>initial load 910/1000; quality density 142.9</p></div>
|
||||
</section>
|
||||
|
||||
<section id="trust" class="twocol">
|
||||
<div class="panel"><h2>信任报告</h2><p>{'scanned_files': 127, 'script_count': 50, 'secret_findings': 0, 'dependency_files': ['requirements-ci.txt'], 'network_script_count': 2, 'interactive_script_count': 0, 'package_sha256': '18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad'}</p></div>
|
||||
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="atlas" class="twocol">
|
||||
<div class="panel"><h2>组合治理</h2><p>12 skills; 5 route collisions; 9 owner gaps; 10 stale</p></div>
|
||||
<div class="panel"><h2>下一动作</h2><p>优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps,再做 registry 或 telemetry。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="release" class="twocol">
|
||||
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked</p></div>
|
||||
<div class="panel"><h2>世界级缺口</h2><p>下一阶段应继续推进 IR-first compiler、registry audit、telemetry drift loop,以及更严格的 governed trust gates。</p></div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
File diff suppressed because it is too large
Load Diff
@@ -2,15 +2,15 @@
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"scanned_files": 124,
|
||||
"script_count": 49,
|
||||
"scanned_files": 127,
|
||||
"script_count": 50,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
"requirements-ci.txt"
|
||||
],
|
||||
"network_script_count": 2,
|
||||
"interactive_script_count": 0,
|
||||
"package_sha256": "d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac"
|
||||
"package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [
|
||||
@@ -275,6 +275,14 @@
|
||||
"uses_network": false,
|
||||
"uses_subprocess": false
|
||||
},
|
||||
{
|
||||
"path": "scripts/render_review_studio.py",
|
||||
"has_argparse": true,
|
||||
"has_main_guard": true,
|
||||
"uses_input": false,
|
||||
"uses_network": false,
|
||||
"uses_subprocess": false
|
||||
},
|
||||
{
|
||||
"path": "scripts/render_review_viewer.py",
|
||||
"has_argparse": true,
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
# Security Trust Report
|
||||
|
||||
- OK: `True`
|
||||
- Scanned files: `124`
|
||||
- Scripts: `49`
|
||||
- Scanned files: `127`
|
||||
- Scripts: `50`
|
||||
- Secret findings: `0`
|
||||
- Network-capable scripts: `2`
|
||||
- Interactive scripts: `0`
|
||||
- Package SHA256: `d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac`
|
||||
- Package SHA256: `18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad`
|
||||
|
||||
## Failures
|
||||
|
||||
@@ -59,6 +59,7 @@
|
||||
| scripts/render_reference_scan.py | True | True | False | False | False |
|
||||
| scripts/render_reference_synthesis.py | True | True | False | False | False |
|
||||
| scripts/render_regression_history.py | True | True | False | False | False |
|
||||
| scripts/render_review_studio.py | True | True | False | False | False |
|
||||
| scripts/render_review_viewer.py | True | True | False | False | False |
|
||||
| scripts/render_skill_overview.py | True | True | False | False | False |
|
||||
| scripts/render_social_preview.py | False | True | False | False | False |
|
||||
|
||||
@@ -13,8 +13,10 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
|
||||
- Trust/Security v0 for secret, dependency, script, trust metadata, and package-integrity checks.
|
||||
- Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, and no-route opportunities.
|
||||
- Bilingual Skill Overview v2 that includes these evidence surfaces.
|
||||
- Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, and release-gate review.
|
||||
- IR-first packaging v0 so adapters carry the platform-neutral semantic contract, parity checks, and IR provenance.
|
||||
|
||||
This is still not the final world-class state. Registry, compiler refactor, telemetry, and Review Studio 2.0 remain open.
|
||||
This is still not the final world-class state. Registry, full target-specific compiler transforms, telemetry, and stricter governed trust gates remain open.
|
||||
|
||||
## Coverage Matrix
|
||||
|
||||
@@ -25,31 +27,31 @@ This is still not the final world-class state. Registry, compiler refactor, tele
|
||||
| Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed |
|
||||
| Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed |
|
||||
| Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed |
|
||||
| Review Studio 2.0 | Current `reports/skill-overview.html`, `reports/review-viewer.html`, and `reports/skill_atlas.html` are separate | partial |
|
||||
| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` | v0 landed |
|
||||
| Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `reports/skill_atlas.html` | v0 landed |
|
||||
| Registry & Distribution | Existing packager, no registry audit/package schema yet | partial |
|
||||
| Registry & Distribution | IR-sourced packager, no registry audit/package schema yet | partial |
|
||||
| Telemetry & Drift | Regression history exists, no adoption or activation telemetry yet | partial |
|
||||
| Compiler from IR | Packager still reads source metadata directly | missing |
|
||||
| Compiler from IR | Packager consumes Skill IR for core semantic adapter fields and keeps frontmatter/interface parity checks | v0 landed |
|
||||
|
||||
## Top Findings
|
||||
|
||||
### 1. Compiler path is not IR-first yet
|
||||
### 1. Compiler path is IR-first v0, but transforms are still shallow
|
||||
|
||||
The new IR captures the capability contract, but `scripts/cross_packager.py` still builds target metadata directly from `SKILL.md` and `agents/interface.yaml`.
|
||||
The packager now reads Skill IR for core semantic fields and emits provenance, schema version, job-to-be-done, semantic contract counts, governance, risk, targets, and semantic parity in each adapter.
|
||||
|
||||
Next move: add `scripts/compile_skill.py` or refactor packager to consume IR for core fields, then assert semantic parity.
|
||||
Next move: add `scripts/compile_skill.py` or split the packager into target-specific transforms so OpenAI, Claude, Agent Skills, VS Code/Copilot, and generic packages can preserve runtime semantics beyond metadata.
|
||||
|
||||
### 2. Output eval is useful but too small
|
||||
### 2. Output eval now meets the governed v0 minimum, but is still static
|
||||
|
||||
The v0 cases prove the runner and scorecard work, but the set is only three static cases. The 2.0 plan calls for library/governed coverage with at least five cases, including boundary, near-neighbor, and real file/output constraints.
|
||||
The v0 cases now cover five scenarios, including near-neighbor and file-backed governed package cases. The next gap is that they are still static text comparisons rather than model-executed holdout runs with timing, tokens, and reviewer adjudication.
|
||||
|
||||
Next move: add holdout output cases and one file-backed fixture case.
|
||||
Next move: add model-executed output eval runs, blind A/B comparison, and one real multi-file fixture.
|
||||
|
||||
### 3. Review Studio is still split
|
||||
### 3. Review Studio is unified, but needs reviewer actions
|
||||
|
||||
The overview report is strong, but the reviewer still has to inspect multiple pages for output eval, conformance, trust, and release evidence.
|
||||
The new Review Studio page aggregates intent, trigger, output, context, runtime, trust, atlas, and release gates. It now exposes current warnings directly: low generic intent-confidence context, trust-script warnings, and portfolio-level Atlas gaps.
|
||||
|
||||
Next move: add a Review Studio 2.0 section or page that aggregates trigger, output, conformance, trust, and release status into one blocking-issue surface.
|
||||
Next move: add reviewer annotations, waived-risk records, and links from each warning to the exact source fix.
|
||||
|
||||
### 4. Multi-skill operation now has v0 coverage, but no telemetry
|
||||
|
||||
@@ -67,17 +69,19 @@ Next move: add stricter governed-mode gates and package hash verification after
|
||||
|
||||
| Gate | Current Result |
|
||||
| --- | --- |
|
||||
| Output Eval | `3` cases, with-skill pass rate `100`, baseline pass rate `0` |
|
||||
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, and boundary coverage |
|
||||
| Runtime Conformance | `5 / 5` targets passing |
|
||||
| Trust | `0` secret findings, `1` pinned dependency file, `2` network-capable scripts flagged as warnings |
|
||||
| Skill Atlas | local workspace scan generated catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview |
|
||||
| Context Budget | initial load remains under the production budget |
|
||||
| Review Studio | `8` gates, `0` blockers, `3` warnings, world-class score `87/100` |
|
||||
| IR-first Packaging | `openai`, `claude`, and `generic` adapters include IR provenance and semantic parity checks |
|
||||
| Context Budget | initial load `910/1000`, under the production budget |
|
||||
| CI | `make ci-test` passed after adding v0 gates |
|
||||
|
||||
## Next Highest-Leverage Moves
|
||||
|
||||
1. Refactor packaging toward IR-first compiler behavior.
|
||||
2. Add Review Studio 2.0 as a single reviewer surface.
|
||||
3. Expand Output Eval Lab from static smoke cases to holdout and file-backed cases.
|
||||
1. Split IR-first packaging into target-specific compiler transforms.
|
||||
2. Add reviewer annotation and waived-risk records to Review Studio.
|
||||
3. Expand Output Eval Lab from static cases to model-executed holdout and blind A/B cases.
|
||||
4. Add registry package schema, package hash, and upgrade audit.
|
||||
5. Connect Skill Atlas with telemetry and drift history.
|
||||
|
||||
@@ -606,7 +606,7 @@
|
||||
<ul class="list"><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">Skill 作者、复用团队和后续 reviewer。</span><span data-lang="en">Skill 作者、复用团队和后续 reviewer。</span></li><li><span data-lang="zh-CN">创建完成后建议先打开 reports/skill-overview.html,再继续扩展包体。</span><span data-lang="en">创建完成后建议先打开 reports/skill-overview.html,再继续扩展包体。</span></li></ul>
|
||||
</aside>
|
||||
</div>
|
||||
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在,是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在,是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 366 个词/字,references 约 11029 个词/字。</span><span data-lang="en">入口约 366 个词/字,references 约 11029 个词/字。</span></small></article></div>
|
||||
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在,是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在,是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 346 个词/字,references 约 11313 个词/字。</span><span data-lang="en">入口约 346 个词/字,references 约 11313 个词/字。</span></small></article></div>
|
||||
</section>
|
||||
|
||||
<section id="overview">
|
||||
@@ -618,7 +618,7 @@
|
||||
<div class="two-col">
|
||||
<article class="panel">
|
||||
<h3><span data-lang="zh-CN">作用定位</span><span data-lang="en">Role</span></h3>
|
||||
<ul class="list"><li><span data-lang="zh-CN">从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill;适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。</span><span data-lang="en">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</span></li><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">交付结果:SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/skill-overview.html</span><span data-lang="en">交付结果:SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/skill-overview.html</span></li></ul>
|
||||
<ul class="list"><li><span data-lang="zh-CN">从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill;适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。</span><span data-lang="en">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</span></li><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">交付结果:SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html</span><span data-lang="en">交付结果:SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/review-studio.html, reports/skill-overview.html</span></li></ul>
|
||||
</article>
|
||||
<figure class="chart-figure" data-chart="flow"><svg viewBox="0 0 620 170" role="img" aria-label="交付流程"><text x="20" y="28" class="chart-title">交付流程</text><path d="M188 93 H248 M398 93 H458" class="chart-line"/><g><rect x="38" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="113" y="99" text-anchor="middle">输入材料</text></g><g><rect x="248" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="323" y="99" text-anchor="middle">Skill 包体</text></g><g><rect x="458" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="533" y="99" text-anchor="middle">可复用能力</text></g></svg><figcaption>交付流程把用户输入、生成的包体和可复用能力放在一条线上。</figcaption></figure>
|
||||
</div>
|
||||
@@ -638,10 +638,10 @@
|
||||
<article class="panel metrics-note">
|
||||
<h3><span data-lang="zh-CN">指标判读</span><span data-lang="en">Reading</span></h3>
|
||||
<p><span data-lang="zh-CN">先看雷达图判断能力短板,再看下方每项分数的证据原因。分数不是装饰数字,必须和本地文件、reports 证据或证据不足提示对应。</span><span data-lang="en">Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.</span></p>
|
||||
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在,是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在,是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 366 个词/字。</span><span data-lang="en">SKILL.md 约 366 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 366 个词/字,references 约 11029 个词/字。</span><span data-lang="en">入口约 366 个词/字,references 约 11029 个词/字。</span></small></li></ol>
|
||||
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在,是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在,是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 346 个词/字。</span><span data-lang="en">SKILL.md 约 346 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 346 个词/字,references 约 11313 个词/字。</span><span data-lang="en">入口约 346 个词/字,references 约 11313 个词/字。</span></small></li></ol>
|
||||
</article>
|
||||
</div>
|
||||
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在,是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在,是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在,便于人工阅读。</span><span data-lang="en">README.md 已存在,便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在,便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在,便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">intent-dialogue.json 已存在。</span><span data-lang="en">intent-dialogue.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 366 个词/字。</span><span data-lang="en">SKILL.md 约 366 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制,可维护性较好。</span><span data-lang="en">入口文件保持克制,可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 366 个词/字,references 约 11029 个词/字。</span><span data-lang="en">入口约 366 个词/字,references 约 11029 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高,建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高,建议压缩入口或拆分 references。</span></li></ul></div></article></div>
|
||||
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在,是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在,是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在,便于人工阅读。</span><span data-lang="en">README.md 已存在,便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在,便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在,便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">intent-dialogue.json 已存在。</span><span data-lang="en">intent-dialogue.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 346 个词/字。</span><span data-lang="en">SKILL.md 约 346 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制,可维护性较好。</span><span data-lang="en">入口文件保持克制,可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 346 个词/字,references 约 11313 个词/字。</span><span data-lang="en">入口约 346 个词/字,references 约 11313 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高,建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高,建议压缩入口或拆分 references。</span></li></ul></div></article></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
@@ -717,7 +717,7 @@
|
||||
<div>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th><th><span data-lang="zh-CN">证据</span><span data-lang="en">Evidence</span></th><th><span data-lang="zh-CN">建议</span><span data-lang="en">Action</span></th></tr></thead>
|
||||
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简,并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR,核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR,核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Output Eval Lab scorecard,可比较 with-skill 与 baseline 输出质量。</span><span data-lang="en">已生成 Output Eval Lab scorecard,可比较 with-skill 与 baseline 输出质量。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强:入口约 366 个词/字,references 约 11029 个词/字。</span><span data-lang="en">上下文成本需要补强:入口约 366 个词/字,references 约 11029 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
|
||||
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简,并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR,核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR,核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Output Eval Lab scorecard,可比较 with-skill 与 baseline 输出质量。</span><span data-lang="en">已生成 Output Eval Lab scorecard,可比较 with-skill 与 baseline 输出质量。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强:入口约 346 个词/字,references 约 11313 个词/字。</span><span data-lang="en">上下文成本需要补强:入口约 346 个词/字,references 约 11313 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
@@ -734,7 +734,7 @@
|
||||
<div>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">风险</span><span data-lang="en">Risk</span></th><th><span data-lang="zh-CN">信号</span><span data-lang="en">Signal</span></th><th><span data-lang="zh-CN">应对</span><span data-lang="en">Response</span></th></tr></thead>
|
||||
<tbody><tr><td><span data-lang="zh-CN">误触发风险</span><span data-lang="en">误触发风险</span></td><td><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">输出漂移风险</span><span data-lang="en">输出漂移风险</span></td><td><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">证据不足风险</span><span data-lang="en">证据不足风险</span></td><td><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">包体膨胀风险</span><span data-lang="en">包体膨胀风险</span></td><td><span data-lang="zh-CN">SKILL.md 约 366 个词/字。</span><span data-lang="en">SKILL.md 约 366 个词/字。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">跨平台迁移风险</span><span data-lang="en">跨平台迁移风险</span></td><td><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr></tbody>
|
||||
<tbody><tr><td><span data-lang="zh-CN">误触发风险</span><span data-lang="en">误触发风险</span></td><td><span data-lang="zh-CN">frontmatter description 已存在,具备基础路由面。</span><span data-lang="en">frontmatter description 已存在,具备基础路由面。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">输出漂移风险</span><span data-lang="en">输出漂移风险</span></td><td><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">证据不足风险</span><span data-lang="en">证据不足风险</span></td><td><span data-lang="zh-CN">已生成 13 / 13 类报告证据。</span><span data-lang="en">已生成 13 / 13 类报告证据。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">包体膨胀风险</span><span data-lang="en">包体膨胀风险</span></td><td><span data-lang="zh-CN">SKILL.md 约 346 个词/字。</span><span data-lang="en">SKILL.md 约 346 个词/字。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr><tr><td><span data-lang="zh-CN">跨平台迁移风险</span><span data-lang="en">跨平台迁移风险</span></td><td><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></td><td><span data-lang="zh-CN">先补证据和边界,再增加包体复杂度。</span><span data-lang="en">先补证据和边界,再增加包体复杂度。</span></td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
@@ -748,7 +748,7 @@
|
||||
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
|
||||
</div>
|
||||
<div class="two-col">
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-1.2" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-2.5" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-3.7" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="34.6 65.4" stroke-dashoffset="-4.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="60.5 39.5" stroke-dashoffset="-39.5" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">250项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-1.2" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-2.4" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="1.2 98.8" stroke-dashoffset="-3.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="34.9 65.1" stroke-dashoffset="-4.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="60.2 39.8" stroke-dashoffset="-39.8" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">255项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
|
||||
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
|
||||
|
||||
+28
-24
@@ -16,6 +16,7 @@
|
||||
"reports/conformance_matrix.md",
|
||||
"reports/security_trust_report.md",
|
||||
"reports/skill_atlas.html",
|
||||
"reports/review-studio.html",
|
||||
"reports/skill-overview.html"
|
||||
],
|
||||
"flow": [
|
||||
@@ -61,7 +62,7 @@
|
||||
"label": "可维护性",
|
||||
"score": 100,
|
||||
"reasons": [
|
||||
"SKILL.md 约 366 个词/字。",
|
||||
"SKILL.md 约 346 个词/字。",
|
||||
"入口文件保持克制,可维护性较好。",
|
||||
"references/ 已承载扩展指导。",
|
||||
"scripts/ 已承载确定性逻辑。",
|
||||
@@ -82,7 +83,7 @@
|
||||
"label": "上下文成本",
|
||||
"score": 42,
|
||||
"reasons": [
|
||||
"入口约 366 个词/字,references 约 11029 个词/字。",
|
||||
"入口约 346 个词/字,references 约 11313 个词/字。",
|
||||
"分数越高代表上下文成本越低。",
|
||||
"上下文成本偏高,建议压缩入口或拆分 references。"
|
||||
]
|
||||
@@ -186,7 +187,7 @@
|
||||
"已生成 Skill Atlas,可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。"
|
||||
],
|
||||
"gaps": [
|
||||
"上下文成本需要补强:入口约 366 个词/字,references 约 11029 个词/字。"
|
||||
"上下文成本需要补强:入口约 346 个词/字,references 约 11313 个词/字。"
|
||||
],
|
||||
"recommendations": [
|
||||
"先改触发边界,再扩展工作流。",
|
||||
@@ -194,7 +195,7 @@
|
||||
"每次升级后重新生成报告并检查分数原因。"
|
||||
],
|
||||
"artifact_design": {
|
||||
"design_system": "evidence frame",
|
||||
"design_system": "review studio",
|
||||
"highlights": [
|
||||
"Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
|
||||
"Name the working directory and required inputs for commands.",
|
||||
@@ -251,7 +252,7 @@
|
||||
"name": "包体膨胀风险",
|
||||
"impact": 2,
|
||||
"probability": 1,
|
||||
"signal": "SKILL.md 约 366 个词/字。",
|
||||
"signal": "SKILL.md 约 346 个词/字。",
|
||||
"response": "先补证据和边界,再增加包体复杂度。"
|
||||
},
|
||||
{
|
||||
@@ -413,28 +414,28 @@
|
||||
"path": "references",
|
||||
"label": "Extended guidance and reusable notes",
|
||||
"kind": "folder",
|
||||
"file_count": 28
|
||||
"file_count": 29
|
||||
},
|
||||
{
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 49
|
||||
"file_count": 50
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
"label": "Trigger and quality checks",
|
||||
"kind": "folder",
|
||||
"file_count": 27
|
||||
"file_count": 28
|
||||
},
|
||||
{
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 142
|
||||
"file_count": 144
|
||||
}
|
||||
],
|
||||
"file_count": 250,
|
||||
"file_count": 255,
|
||||
"folder_count": 4,
|
||||
"distribution": [
|
||||
{
|
||||
@@ -455,19 +456,19 @@
|
||||
},
|
||||
{
|
||||
"label": "references",
|
||||
"value": 28
|
||||
"value": 29
|
||||
},
|
||||
{
|
||||
"label": "scripts",
|
||||
"value": 49
|
||||
"value": 50
|
||||
},
|
||||
{
|
||||
"label": "evals",
|
||||
"value": 27
|
||||
"value": 28
|
||||
},
|
||||
{
|
||||
"label": "reports",
|
||||
"value": 142
|
||||
"value": 144
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -582,25 +583,25 @@
|
||||
"path": "references",
|
||||
"label": "Extended guidance and reusable notes",
|
||||
"kind": "folder",
|
||||
"file_count": 28
|
||||
"file_count": 29
|
||||
},
|
||||
{
|
||||
"path": "scripts",
|
||||
"label": "Deterministic helpers or local tooling",
|
||||
"kind": "folder",
|
||||
"file_count": 49
|
||||
"file_count": 50
|
||||
},
|
||||
{
|
||||
"path": "evals",
|
||||
"label": "Trigger and quality checks",
|
||||
"kind": "folder",
|
||||
"file_count": 27
|
||||
"file_count": 28
|
||||
},
|
||||
{
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 142
|
||||
"file_count": 144
|
||||
}
|
||||
],
|
||||
"strengths": [
|
||||
@@ -704,10 +705,13 @@
|
||||
"schema_version": "2.0.0",
|
||||
"target_count": 4,
|
||||
"trigger_samples": 8,
|
||||
"output_eval_cases": 2
|
||||
"output_eval_cases": 3
|
||||
},
|
||||
"output_quality": {
|
||||
"case_count": 3,
|
||||
"case_count": 5,
|
||||
"file_backed_case_count": 1,
|
||||
"near_neighbor_case_count": 1,
|
||||
"boundary_case_count": 1,
|
||||
"baseline_pass_rate": 0.0,
|
||||
"with_skill_pass_rate": 100.0,
|
||||
"delta": 100.0,
|
||||
@@ -721,15 +725,15 @@
|
||||
"fail_count": 0
|
||||
},
|
||||
"trust_security": {
|
||||
"scanned_files": 124,
|
||||
"script_count": 49,
|
||||
"scanned_files": 127,
|
||||
"script_count": 50,
|
||||
"secret_findings": 0,
|
||||
"dependency_files": [
|
||||
"requirements-ci.txt"
|
||||
],
|
||||
"network_script_count": 2,
|
||||
"interactive_script_count": 0,
|
||||
"package_sha256": "d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac"
|
||||
"package_sha256": "18618ecee3683f7488ae4a65a9800c428ee15c6539bbcd367910c57d84ac77ad"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -745,7 +749,7 @@
|
||||
"Borrow the discipline of defining what the skill should not own before growing the package."
|
||||
],
|
||||
"artifact_design": {
|
||||
"design_system": "evidence frame",
|
||||
"design_system": "review studio",
|
||||
"highlights": [
|
||||
"Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
|
||||
"Name the working directory and required inputs for commands.",
|
||||
|
||||
@@ -62,6 +62,7 @@
|
||||
"scripts/render_reference_scan.py",
|
||||
"scripts/render_reference_synthesis.py",
|
||||
"scripts/render_regression_history.py",
|
||||
"scripts/render_review_studio.py",
|
||||
"scripts/render_review_viewer.py",
|
||||
"scripts/render_skill_overview.py",
|
||||
"scripts/render_social_preview.py",
|
||||
@@ -100,6 +101,7 @@
|
||||
"references/reference-scan.md",
|
||||
"references/regression-cause-taxonomy.md",
|
||||
"references/resource-boundaries.md",
|
||||
"references/review-studio-method.md",
|
||||
"references/runtime-conformance-method.md",
|
||||
"references/skill-archetypes.md",
|
||||
"references/skill-atlas-method.md",
|
||||
|
||||
@@ -91,7 +91,9 @@ def should_skip(path: Path, root: Path) -> bool:
|
||||
rel = path.relative_to(root)
|
||||
except ValueError:
|
||||
return True
|
||||
return any(part in IGNORE_PARTS for part in rel.parts)
|
||||
if any(part in IGNORE_PARTS for part in rel.parts):
|
||||
return True
|
||||
return len(rel.parts) >= 2 and rel.parts[0] == "tests" and rel.parts[1].startswith("tmp")
|
||||
|
||||
|
||||
def find_skill_dirs(workspace_root: Path) -> list[Path]:
|
||||
@@ -136,7 +138,8 @@ def resource_names(skill_dir: Path) -> list[str]:
|
||||
if not target.exists():
|
||||
continue
|
||||
for path in sorted(target.rglob("*")):
|
||||
if any(part in IGNORE_PARTS for part in path.relative_to(skill_dir).parts):
|
||||
rel = path.relative_to(skill_dir)
|
||||
if any(part in IGNORE_PARTS for part in rel.parts):
|
||||
continue
|
||||
if path.suffix in {".pyc", ".pyo"}:
|
||||
continue
|
||||
|
||||
@@ -39,6 +39,7 @@ DEFAULT_TARGETS = [
|
||||
"trust-check",
|
||||
"skill-atlas-check",
|
||||
"review-viewer-check",
|
||||
"review-studio-check",
|
||||
"feedback-check",
|
||||
"baseline-compare-check",
|
||||
"reference-scan-check",
|
||||
|
||||
+155
-2
@@ -7,6 +7,20 @@ from pathlib import Path
|
||||
import yaml
|
||||
|
||||
|
||||
def display_path(path: Path, root: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(root.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def read_json(path: Path) -> dict:
|
||||
if not path.exists():
|
||||
return {}
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def read_simple_yaml(path: Path) -> dict:
|
||||
return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
@@ -31,6 +45,23 @@ def read_interface(skill_dir: Path) -> dict:
|
||||
return raw
|
||||
|
||||
|
||||
def find_skill_ir(skill_dir: Path, name: str) -> tuple[dict, str]:
|
||||
candidates = [
|
||||
skill_dir / "reports" / "skill-ir.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{name}.json",
|
||||
skill_dir / "skill-ir" / "examples" / f"{skill_dir.name}.json",
|
||||
]
|
||||
seen = set()
|
||||
for path in candidates:
|
||||
if path in seen:
|
||||
continue
|
||||
seen.add(path)
|
||||
payload = read_json(path)
|
||||
if payload:
|
||||
return payload, display_path(path, skill_dir)
|
||||
return {}, "frontmatter-fallback"
|
||||
|
||||
|
||||
def require_fields(payload: dict, fields: list[str], label: str) -> None:
|
||||
missing = [field for field in fields if not payload.get(field)]
|
||||
if missing:
|
||||
@@ -46,8 +77,99 @@ def require_target_degradation(
|
||||
raise ValueError(f"Missing degradation entries for targets: {', '.join(missing)}")
|
||||
|
||||
|
||||
def count_list(payload: dict, key: str) -> int:
|
||||
value = payload.get(key, [])
|
||||
return len(value) if isinstance(value, list) else 0
|
||||
|
||||
|
||||
def resource_counts(resources: dict) -> dict:
|
||||
return {
|
||||
"references": count_list(resources, "references"),
|
||||
"scripts": count_list(resources, "scripts"),
|
||||
"assets": count_list(resources, "assets"),
|
||||
"reports": count_list(resources, "reports"),
|
||||
}
|
||||
|
||||
|
||||
def eval_counts(eval_plan: dict) -> dict:
|
||||
return {
|
||||
"trigger": count_list(eval_plan, "trigger"),
|
||||
"output": count_list(eval_plan, "output"),
|
||||
"adversarial": count_list(eval_plan, "adversarial"),
|
||||
"baseline": 1 if eval_plan.get("baseline") else 0,
|
||||
}
|
||||
|
||||
|
||||
def build_semantic_contract(
|
||||
*,
|
||||
skill_dir: Path,
|
||||
platform: str,
|
||||
frontmatter: dict,
|
||||
interface: dict,
|
||||
compatibility: dict,
|
||||
manifest: dict,
|
||||
ir: dict,
|
||||
ir_source: str,
|
||||
) -> dict:
|
||||
trigger_surface = ir.get("trigger_surface", {}) if isinstance(ir.get("trigger_surface"), dict) else {}
|
||||
workflow = ir.get("workflow", {}) if isinstance(ir.get("workflow"), dict) else {}
|
||||
resources = ir.get("resources", {}) if isinstance(ir.get("resources"), dict) else {}
|
||||
eval_plan = ir.get("eval_plan", {}) if isinstance(ir.get("eval_plan"), dict) else {}
|
||||
|
||||
frontmatter_name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name)
|
||||
frontmatter_description = str(frontmatter.get("description") or "")
|
||||
ir_name = str(ir.get("name") or "") if ir else ""
|
||||
ir_description = str(trigger_surface.get("description") or "") if ir else ""
|
||||
name = ir_name or frontmatter_name
|
||||
description = ir_description or frontmatter_description
|
||||
job = str(ir.get("job_to_be_done") or description)
|
||||
targets = ir.get("targets") if isinstance(ir.get("targets"), list) else compatibility.get("adapter_targets", [])
|
||||
target_values = [str(item) for item in targets]
|
||||
adapter_targets = [str(item) for item in compatibility.get("adapter_targets", [])]
|
||||
|
||||
semantic_contract = {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"job_to_be_done": job,
|
||||
"trigger_description": description,
|
||||
"should_trigger_count": count_list(trigger_surface, "should_trigger"),
|
||||
"should_not_trigger_count": count_list(trigger_surface, "should_not_trigger"),
|
||||
"edge_case_count": count_list(trigger_surface, "edge_cases"),
|
||||
"workflow_step_count": count_list(workflow, "steps"),
|
||||
"decision_point_count": count_list(workflow, "decision_points"),
|
||||
"failure_mode_count": count_list(workflow, "failure_modes"),
|
||||
"resource_counts": resource_counts(resources),
|
||||
"eval_counts": eval_counts(eval_plan),
|
||||
"risk": ir.get("risk", {}) if isinstance(ir.get("risk"), dict) else {},
|
||||
"governance": ir.get("governance", {}) if isinstance(ir.get("governance"), dict) else {},
|
||||
"targets": target_values,
|
||||
"source_files_count": count_list(ir, "source_files") if ir else 0,
|
||||
}
|
||||
semantic_parity = {
|
||||
"source": "skill-ir" if ir else "frontmatter-fallback",
|
||||
"ir_source": ir_source,
|
||||
"name_matches_ir": bool(ir) and frontmatter_name == name,
|
||||
"description_matches_ir": bool(ir) and frontmatter_description == description,
|
||||
"platform_declared_in_ir": platform in target_values
|
||||
or (platform == "generic" and "agent-skills-compatible" in target_values),
|
||||
"platform_declared_in_interface": platform in adapter_targets,
|
||||
"display_name_present": bool(interface.get("display_name")),
|
||||
"default_prompt_present": bool(interface.get("default_prompt")),
|
||||
}
|
||||
return {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"job_to_be_done": job,
|
||||
"ir_source": ir_source,
|
||||
"ir_schema_version": str(ir.get("schema_version") or "none"),
|
||||
"semantic_contract": semantic_contract,
|
||||
"semantic_parity": semantic_parity,
|
||||
}
|
||||
|
||||
|
||||
def build_manifest(skill_dir: Path, platform: str) -> dict:
|
||||
frontmatter = read_frontmatter(skill_dir / "SKILL.md")
|
||||
manifest = read_json(skill_dir / "manifest.json")
|
||||
interface_doc = read_interface(skill_dir)
|
||||
interface = interface_doc.get("interface", {})
|
||||
compatibility = interface_doc.get("compatibility", {})
|
||||
@@ -66,12 +188,28 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
|
||||
"compatibility.trust",
|
||||
)
|
||||
require_target_degradation(degradation, compatibility.get("adapter_targets", []))
|
||||
ir, ir_source = find_skill_ir(skill_dir, str(frontmatter.get("name") or manifest.get("name") or skill_dir.name))
|
||||
semantic = build_semantic_contract(
|
||||
skill_dir=skill_dir,
|
||||
platform=platform,
|
||||
frontmatter=frontmatter,
|
||||
interface=interface,
|
||||
compatibility=compatibility,
|
||||
manifest=manifest,
|
||||
ir=ir,
|
||||
ir_source=ir_source,
|
||||
)
|
||||
return {
|
||||
"name": frontmatter.get("name", skill_dir.name),
|
||||
"description": frontmatter.get("description", ""),
|
||||
"name": semantic["name"],
|
||||
"description": semantic["description"],
|
||||
"version": frontmatter.get("version", "1.0.0"),
|
||||
"platform": platform,
|
||||
"skill_root": skill_dir.name,
|
||||
"job_to_be_done": semantic["job_to_be_done"],
|
||||
"ir_source": semantic["ir_source"],
|
||||
"ir_schema_version": semantic["ir_schema_version"],
|
||||
"semantic_contract": semantic["semantic_contract"],
|
||||
"semantic_parity": semantic["semantic_parity"],
|
||||
"display_name": interface.get("display_name", skill_dir.name),
|
||||
"short_description": interface.get("short_description", ""),
|
||||
"default_prompt": interface.get("default_prompt", ""),
|
||||
@@ -108,6 +246,11 @@ PLATFORM_CONTRACTS = {
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
@@ -135,6 +278,11 @@ PLATFORM_CONTRACTS = {
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
@@ -162,6 +310,11 @@ PLATFORM_CONTRACTS = {
|
||||
"display_name",
|
||||
"short_description",
|
||||
"default_prompt",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity",
|
||||
"canonical_metadata",
|
||||
"canonical_format",
|
||||
"activation_mode",
|
||||
|
||||
+21
-7
@@ -15,6 +15,7 @@ from render_output_risk_profile import render_output_risk_profile
|
||||
from render_prompt_quality_profile import render_prompt_quality_profile
|
||||
from render_reference_scan import parse_reference, render_reference_scan
|
||||
from render_reference_synthesis import render_reference_synthesis
|
||||
from render_review_studio import render_review_studio
|
||||
from render_review_viewer import render_review_viewer
|
||||
from render_skill_overview import render_skill_overview
|
||||
from render_system_model import render_system_model
|
||||
@@ -69,13 +70,14 @@ README_TEMPLATE = """# {title}
|
||||
5. Open `reports/reference-synthesis.md` to see the GitHub benchmarks plus curated official, research, and principle tracks in one place.
|
||||
6. Follow the workflow steps in `SKILL.md`.
|
||||
7. Check `reports/skill-overview.html` for the generated bilingual HTML skill audit report: overview, metrics, capability profile, principle, contract, quality, risk, assets, and iteration roadmap. It defaults to Simplified Chinese and includes an English switch in the top right.
|
||||
8. Open `reports/review-viewer.html` for a compact visual review of the package.
|
||||
9. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks.
|
||||
10. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns.
|
||||
11. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix.
|
||||
12. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging.
|
||||
13. Review `reports/iteration-directions.md` for the three most valuable next moves.
|
||||
14. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes.
|
||||
8. Open `reports/review-studio.html` for the one-page Review Studio 2.0 gate view.
|
||||
9. Open `reports/review-viewer.html` for a compact visual review of the package.
|
||||
10. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks.
|
||||
11. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns.
|
||||
12. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix.
|
||||
13. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging.
|
||||
14. Review `reports/iteration-directions.md` for the three most valuable next moves.
|
||||
15. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes.
|
||||
|
||||
## Honest Boundaries
|
||||
|
||||
@@ -99,6 +101,7 @@ README_TEMPLATE = """# {title}
|
||||
- `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points
|
||||
- `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance
|
||||
- `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap
|
||||
- `reports/review-studio.html`: Review Studio 2.0 gate page for intent, trigger, output eval, context, runtime conformance, trust, atlas, and release readiness
|
||||
- `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps
|
||||
- `reports/iteration-directions.md`: the top three next iteration directions
|
||||
"""
|
||||
@@ -211,14 +214,17 @@ def build_report_view(artifacts: dict) -> dict:
|
||||
html_report = artifacts.get("skill_overview_html", "")
|
||||
json_report = artifacts.get("skill_overview_json", "")
|
||||
system_model = artifacts.get("system_model_md", "")
|
||||
review_studio = artifacts.get("review_studio_html", "")
|
||||
return {
|
||||
"title": "Skill 总结报告",
|
||||
"html_report": html_report,
|
||||
"json_report": json_report,
|
||||
"system_model": system_model,
|
||||
"review_studio": review_studio,
|
||||
"message": (
|
||||
f"Skill 已创建完成。建议先打开总结报告:{html_report}。"
|
||||
"它会展示这个 Skill 的概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线;"
|
||||
f"然后打开 Review Studio 2.0:{review_studio},检查意图、触发、输出评测、运行一致性、信任和发布闸门。"
|
||||
"报告默认使用中文简体,右上角可以切换英文版。"
|
||||
),
|
||||
"next_action": "Open reports/skill-overview.html before editing more files.",
|
||||
@@ -243,6 +249,11 @@ def render_skill_ir(root: Path) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def absolute_skill_artifact(root: Path, value: str) -> str:
|
||||
path = Path(value)
|
||||
return str(path if path.is_absolute() else root / path)
|
||||
|
||||
|
||||
def dedupe_references(references: list[dict]) -> list[dict]:
|
||||
seen = set()
|
||||
deduped = []
|
||||
@@ -328,6 +339,7 @@ def initialize_skill(
|
||||
iteration_directions = render_iteration_directions(root)
|
||||
overview = render_skill_overview(root)
|
||||
review_viewer = render_review_viewer(root)
|
||||
review_studio = render_review_studio(root)
|
||||
artifacts = {
|
||||
"readme": str(root / "README.md"),
|
||||
"manifest": str(root / "manifest.json"),
|
||||
@@ -355,6 +367,8 @@ def initialize_skill(
|
||||
"iteration_directions_json": iteration_directions["artifacts"]["json"],
|
||||
"review_viewer_html": review_viewer["artifacts"]["html"],
|
||||
"review_viewer_json": review_viewer["artifacts"]["json"],
|
||||
"review_studio_html": absolute_skill_artifact(root, review_studio["artifacts"]["html"]),
|
||||
"review_studio_json": absolute_skill_artifact(root, review_studio["artifacts"]["json"]),
|
||||
}
|
||||
if benchmark_scan is not None:
|
||||
artifacts["github_benchmark_scan_md"] = benchmark_scan["artifacts"]["markdown"]
|
||||
|
||||
@@ -17,6 +17,17 @@ def load_yaml(path: Path) -> dict:
|
||||
return yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
|
||||
|
||||
def find_ir(root: Path) -> tuple[dict, str]:
|
||||
candidates = [
|
||||
root / "reports" / "skill-ir.json",
|
||||
root / "skill-ir" / "examples" / f"{root.name}.json",
|
||||
]
|
||||
for path in candidates:
|
||||
if path.exists():
|
||||
return load_json(path), str(path.relative_to(root))
|
||||
return {}, "missing"
|
||||
|
||||
|
||||
def band_for(score: int) -> str:
|
||||
if score >= 97:
|
||||
return "world-class"
|
||||
@@ -39,9 +50,12 @@ def build_report(root: Path) -> dict:
|
||||
contracts_doc = (root / "references" / "packaging-contracts.md").exists()
|
||||
matrix_doc = (root / "references" / "platform-capability-matrix.md").exists()
|
||||
snapshot_files = list((root / "tests" / "snapshots").glob("*_adapter.json"))
|
||||
ir, ir_source = find_ir(root)
|
||||
has_neutral_metadata = (root / "SKILL.md").exists() and (root / "agents" / "interface.yaml").exists()
|
||||
has_skill_ir = ir.get("schema_version") == "2.0.0" and bool(ir.get("job_to_be_done"))
|
||||
|
||||
breakdown = {
|
||||
"canonical_neutral_source": 15 if (root / "SKILL.md").exists() and (root / "agents" / "interface.yaml").exists() else 0,
|
||||
"platform_neutral_semantic_source": 15 if has_neutral_metadata and has_skill_ir else 8 if has_neutral_metadata else 0,
|
||||
"adapter_target_coverage": 15 if len(targets) >= 3 else 10 if len(targets) >= 2 else 5 if len(targets) >= 1 else 0,
|
||||
"activation_metadata": 10
|
||||
if activation.get("mode") and activation.get("paths") is not None
|
||||
@@ -83,6 +97,8 @@ def build_report(root: Path) -> dict:
|
||||
"remote_inline_execution": trust.get("remote_inline_execution"),
|
||||
"degradation_coverage": sum(1 for target in targets if degradation.get(target)),
|
||||
"snapshot_count": len(snapshot_files),
|
||||
"ir_source": ir_source,
|
||||
"ir_schema_version": ir.get("schema_version", "missing"),
|
||||
},
|
||||
"breakdown": breakdown,
|
||||
}
|
||||
@@ -100,6 +116,8 @@ def render_markdown(report: dict) -> str:
|
||||
f"- shell: `{report['summary']['shell']}`",
|
||||
f"- trust level: `{report['summary']['trust_level']}`",
|
||||
f"- remote inline execution: `{report['summary']['remote_inline_execution']}`",
|
||||
f"- Skill IR source: `{report['summary']['ir_source']}`",
|
||||
f"- Skill IR schema: `{report['summary']['ir_schema_version']}`",
|
||||
"",
|
||||
"| Component | Score |",
|
||||
"| --- | ---: |",
|
||||
|
||||
@@ -0,0 +1,720 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
try:
|
||||
import yaml
|
||||
except ImportError: # pragma: no cover
|
||||
yaml = None
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def load_yaml(path: Path) -> dict[str, Any]:
|
||||
if not path.exists() or yaml is None:
|
||||
return {}
|
||||
payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
|
||||
|
||||
def parse_frontmatter(path: Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {}
|
||||
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
if not lines or lines[0].strip() != "---":
|
||||
return {}
|
||||
try:
|
||||
end_index = lines[1:].index("---") + 1
|
||||
except ValueError:
|
||||
return {}
|
||||
text = "\n".join(lines[1:end_index])
|
||||
if yaml is not None:
|
||||
payload = yaml.safe_load(text) or {}
|
||||
return payload if isinstance(payload, dict) else {}
|
||||
data = {}
|
||||
for line in text.splitlines():
|
||||
if ":" in line:
|
||||
key, value = line.split(":", 1)
|
||||
data[key.strip()] = value.strip().strip('"')
|
||||
return data
|
||||
|
||||
|
||||
def display_path(skill_dir: Path, path: Path) -> str:
|
||||
try:
|
||||
return str(path.resolve().relative_to(skill_dir.resolve()))
|
||||
except ValueError:
|
||||
try:
|
||||
return str(path.resolve().relative_to(ROOT.resolve()))
|
||||
except ValueError:
|
||||
return str(path.resolve())
|
||||
|
||||
|
||||
def link_from(output_html: Path, target: Path) -> str:
|
||||
return os.path.relpath(target.resolve(), output_html.parent.resolve())
|
||||
|
||||
|
||||
def report_link(output_html: Path, skill_dir: Path, rel_path: str) -> str:
|
||||
return link_from(output_html, skill_dir / rel_path)
|
||||
|
||||
|
||||
def gate(key: str, label: str, status: str, detail: str, evidence: str, link: str = "") -> dict[str, str]:
|
||||
return {
|
||||
"key": key,
|
||||
"label": label,
|
||||
"status": status,
|
||||
"detail": detail,
|
||||
"evidence": evidence,
|
||||
"link": link,
|
||||
}
|
||||
|
||||
|
||||
def status_label(status: str) -> str:
|
||||
return {"pass": "通过", "warn": "关注", "block": "阻断"}.get(status, status)
|
||||
|
||||
|
||||
def add_blockers_from_gate(gates: list[dict[str, str]]) -> tuple[list[dict[str, str]], list[dict[str, str]]]:
|
||||
blockers = [item for item in gates if item["status"] == "block"]
|
||||
warnings = [item for item in gates if item["status"] == "warn"]
|
||||
return blockers, warnings
|
||||
|
||||
|
||||
def target_maturity(skill_dir: Path, overview: dict[str, Any]) -> str:
|
||||
manifest = load_json(skill_dir / "manifest.json")
|
||||
if manifest.get("maturity_tier"):
|
||||
return str(manifest["maturity_tier"])
|
||||
metadata = overview.get("metadata", {}) if isinstance(overview, dict) else {}
|
||||
if metadata.get("maturity_tier"):
|
||||
return str(metadata["maturity_tier"])
|
||||
return "scaffold"
|
||||
|
||||
|
||||
def min_output_cases(maturity: str) -> int:
|
||||
if maturity in {"library", "governed"}:
|
||||
return 5
|
||||
if maturity == "production":
|
||||
return 3
|
||||
return 1
|
||||
|
||||
|
||||
def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
overview = data["overview"]
|
||||
maturity = target_maturity(skill_dir, overview)
|
||||
gates: list[dict[str, str]] = []
|
||||
|
||||
intent = data["intent_confidence"]
|
||||
intent_score = int(intent.get("score", 0) or 0)
|
||||
intent_status = "pass" if intent.get("gate_passed") or intent_score >= 75 else "warn"
|
||||
gates.append(
|
||||
gate(
|
||||
"intent-canvas",
|
||||
"意图画布",
|
||||
intent_status,
|
||||
f"intent confidence {intent_score}/100; {intent.get('recommended_action', 'review current intent frame')}",
|
||||
"reports/intent-confidence.json",
|
||||
report_link(output_html, skill_dir, "reports/intent-confidence.md"),
|
||||
)
|
||||
)
|
||||
|
||||
route = data["route_scorecard"]
|
||||
route_summary = route.get("summary", {})
|
||||
misroutes = int(route_summary.get("misroute_count", len(route.get("misroutes", []))) or 0)
|
||||
ambiguous = int(route_summary.get("ambiguous_case_count", len(route.get("ambiguous_cases", []))) or 0)
|
||||
if not route:
|
||||
route_status = "warn"
|
||||
route_detail = "route scorecard is missing; run route-scorecard before release review"
|
||||
else:
|
||||
route_status = "block" if misroutes else ("warn" if ambiguous else "pass")
|
||||
route_detail = f"{route_summary.get('total_cases', 0)} trigger cases; {misroutes} misroutes; {ambiguous} ambiguous"
|
||||
gates.append(
|
||||
gate(
|
||||
"trigger-lab",
|
||||
"触发实验",
|
||||
route_status,
|
||||
route_detail,
|
||||
"reports/route_scorecard.json",
|
||||
report_link(output_html, skill_dir, "reports/route_scorecard.md"),
|
||||
)
|
||||
)
|
||||
|
||||
output = data["output_quality"]
|
||||
output_summary = output.get("summary", {})
|
||||
required_cases = min_output_cases(maturity)
|
||||
case_count = int(output_summary.get("case_count", 0) or 0)
|
||||
file_backed = int(output_summary.get("file_backed_case_count", 0) or 0)
|
||||
near_neighbor = int(output_summary.get("near_neighbor_case_count", 0) or 0)
|
||||
boundary = int(output_summary.get("boundary_case_count", 0) or 0)
|
||||
output_blocked = not output.get("ok", False) or not output_summary.get("gate_pass", False) or case_count < required_cases
|
||||
output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0
|
||||
if not output:
|
||||
output_status = "warn"
|
||||
output_detail = "output eval scorecard is missing; generate it before production review"
|
||||
else:
|
||||
output_status = "block" if output_blocked else ("warn" if output_warn else "pass")
|
||||
output_detail = (
|
||||
f"{case_count}/{required_cases} cases; with-skill {output_summary.get('with_skill_pass_rate', 0)}; "
|
||||
f"baseline {output_summary.get('baseline_pass_rate', 0)}; file-backed {file_backed}; near-neighbor {near_neighbor}"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"output-lab",
|
||||
"输出实验",
|
||||
output_status,
|
||||
output_detail,
|
||||
"reports/output_quality_scorecard.json",
|
||||
report_link(output_html, skill_dir, "reports/output_quality_scorecard.md"),
|
||||
)
|
||||
)
|
||||
|
||||
context = data["context_budget"]
|
||||
context_stats = context.get("stats", {})
|
||||
context_status = "pass" if context.get("ok") else "block"
|
||||
if context.get("warnings"):
|
||||
context_status = "warn" if context_status == "pass" else context_status
|
||||
if not context:
|
||||
context_status = "warn"
|
||||
context_detail = (
|
||||
f"initial load {context_stats.get('estimated_initial_load_tokens', 'n/a')}/"
|
||||
f"{context_stats.get('context_budget_limit', 'n/a')}; quality density {context_stats.get('quality_density', 'n/a')}"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"context-budget",
|
||||
"上下文",
|
||||
context_status,
|
||||
context_detail,
|
||||
"reports/context_budget.json",
|
||||
report_link(output_html, skill_dir, "reports/context_budget.md"),
|
||||
)
|
||||
)
|
||||
|
||||
conformance = data["conformance"]
|
||||
conformance_summary = conformance.get("summary", {})
|
||||
fail_count = int(conformance_summary.get("fail_count", 0) or 0)
|
||||
if not conformance:
|
||||
conformance_status = "warn"
|
||||
conformance_detail = "runtime conformance matrix is missing"
|
||||
else:
|
||||
conformance_status = "block" if fail_count else "pass"
|
||||
conformance_detail = f"{conformance_summary.get('pass_count', 0)} / {conformance_summary.get('target_count', 0)} targets pass"
|
||||
gates.append(
|
||||
gate(
|
||||
"runtime-matrix",
|
||||
"运行矩阵",
|
||||
conformance_status,
|
||||
conformance_detail,
|
||||
"reports/conformance_matrix.json",
|
||||
report_link(output_html, skill_dir, "reports/conformance_matrix.md"),
|
||||
)
|
||||
)
|
||||
|
||||
trust = data["trust"]
|
||||
trust_summary = trust.get("summary", {})
|
||||
if not trust:
|
||||
trust_status = "warn"
|
||||
trust_detail = "security trust report is missing"
|
||||
else:
|
||||
trust_status = "block" if trust.get("failures") else ("warn" if trust.get("warnings") else "pass")
|
||||
trust_detail = (
|
||||
f"{trust_summary.get('secret_findings', 0)} secrets; "
|
||||
f"{trust_summary.get('script_count', 0)} scripts; "
|
||||
f"{trust_summary.get('network_script_count', 0)} network-capable scripts"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"trust-report",
|
||||
"信任报告",
|
||||
trust_status,
|
||||
trust_detail,
|
||||
"reports/security_trust_report.json",
|
||||
report_link(output_html, skill_dir, "reports/security_trust_report.md"),
|
||||
)
|
||||
)
|
||||
|
||||
atlas = data["atlas"]
|
||||
atlas_summary = atlas.get("summary", {})
|
||||
atlas_issues = int(atlas_summary.get("route_collision_count", 0) or 0) + int(atlas_summary.get("owner_gap_count", 0) or 0)
|
||||
if not atlas:
|
||||
atlas_status = "warn"
|
||||
atlas_detail = "skill atlas is missing; portfolio-level conflicts are unknown"
|
||||
else:
|
||||
atlas_status = "warn" if atlas_issues else "pass"
|
||||
atlas_detail = (
|
||||
f"{atlas_summary.get('skill_count', 0)} skills; "
|
||||
f"{atlas_summary.get('route_collision_count', 0)} route collisions; "
|
||||
f"{atlas_summary.get('owner_gap_count', 0)} owner gaps; "
|
||||
f"{atlas_summary.get('stale_count', 0)} stale"
|
||||
)
|
||||
gates.append(
|
||||
gate(
|
||||
"skill-atlas",
|
||||
"组合治理",
|
||||
atlas_status,
|
||||
atlas_detail,
|
||||
"reports/skill_atlas.json",
|
||||
report_link(output_html, skill_dir, "reports/skill_atlas.html"),
|
||||
)
|
||||
)
|
||||
|
||||
promotion = data["promotion"]
|
||||
migration_path = ROOT / "docs" / "migration-v2.md"
|
||||
if promotion:
|
||||
promotion_summary = promotion.get("summary", {})
|
||||
blocked = int(promotion_summary.get("blocked", 0) or 0)
|
||||
release_status = "block" if blocked else "pass"
|
||||
release_detail = f"{promotion_summary.get('promote', 0)} promote; {promotion_summary.get('keep_current', 0)} keep current; {blocked} blocked"
|
||||
else:
|
||||
release_status = "warn"
|
||||
release_detail = "promotion decisions are missing; release notes need reviewer confirmation"
|
||||
gates.append(
|
||||
gate(
|
||||
"release-notes",
|
||||
"发布路线",
|
||||
release_status,
|
||||
release_detail,
|
||||
"reports/promotion_decisions.json + docs/migration-v2.md",
|
||||
report_link(output_html, skill_dir, "reports/promotion_decisions.md") if promotion else str(migration_path),
|
||||
)
|
||||
)
|
||||
|
||||
return gates
|
||||
|
||||
|
||||
def weighted_score(gates: list[dict[str, str]]) -> int:
|
||||
weights = {
|
||||
"trigger-lab": 15,
|
||||
"output-lab": 20,
|
||||
"context-budget": 10,
|
||||
"runtime-matrix": 10,
|
||||
"trust-report": 10,
|
||||
"skill-atlas": 10,
|
||||
"release-notes": 10,
|
||||
"intent-canvas": 10,
|
||||
}
|
||||
earned = 0.0
|
||||
total = 0.0
|
||||
for item in gates:
|
||||
weight = weights.get(item["key"], 5)
|
||||
total += weight
|
||||
if item["status"] == "pass":
|
||||
earned += weight
|
||||
elif item["status"] == "warn":
|
||||
earned += weight * 0.6
|
||||
return int(round(earned / total * 100)) if total else 0
|
||||
|
||||
|
||||
def evidence_paths(skill_dir: Path) -> dict[str, str]:
|
||||
rels = {
|
||||
"skill_overview": "reports/skill-overview.html",
|
||||
"review_viewer": "reports/review-viewer.html",
|
||||
"output_eval": "reports/output_quality_scorecard.md",
|
||||
"runtime_conformance": "reports/conformance_matrix.md",
|
||||
"trust_report": "reports/security_trust_report.md",
|
||||
"skill_atlas": "reports/skill_atlas.html",
|
||||
"migration": "docs/migration-v2.md",
|
||||
"skill_ir": "reports/skill-ir.json",
|
||||
}
|
||||
return {key: rel for key, rel in rels.items() if (skill_dir / rel).exists() or (ROOT / rel).exists()}
|
||||
|
||||
|
||||
def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]:
|
||||
reports = skill_dir / "reports"
|
||||
return {
|
||||
"overview": load_json(reports / "skill-overview.json"),
|
||||
"intent_confidence": load_json(reports / "intent-confidence.json"),
|
||||
"intent_dialogue": load_json(reports / "intent-dialogue.json"),
|
||||
"route_scorecard": load_json(reports / "route_scorecard.json"),
|
||||
"output_quality": load_json(reports / "output_quality_scorecard.json"),
|
||||
"conformance": load_json(reports / "conformance_matrix.json"),
|
||||
"trust": load_json(reports / "security_trust_report.json"),
|
||||
"context_budget": load_json(reports / "context_budget.json"),
|
||||
"promotion": load_json(reports / "promotion_decisions.json"),
|
||||
"atlas": load_json(reports / "skill_atlas.json"),
|
||||
"manifest": load_json(skill_dir / "manifest.json"),
|
||||
"frontmatter": parse_frontmatter(skill_dir / "SKILL.md"),
|
||||
"interface": load_yaml(skill_dir / "agents" / "interface.yaml"),
|
||||
}
|
||||
|
||||
|
||||
def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]:
|
||||
overview = data["overview"]
|
||||
output = data["output_quality"].get("summary", {})
|
||||
conformance = data["conformance"].get("summary", {})
|
||||
trust = data["trust"].get("summary", {})
|
||||
atlas = data["atlas"].get("summary", {})
|
||||
cards = [
|
||||
{
|
||||
"label": "Skill IR",
|
||||
"value": str(overview.get("skill_ir", {}).get("schema_version", "missing")),
|
||||
"detail": f"{overview.get('skill_ir', {}).get('target_count', 0)} targets in platform-neutral contract",
|
||||
},
|
||||
{
|
||||
"label": "Output Delta",
|
||||
"value": str(output.get("delta", "n/a")),
|
||||
"detail": f"{output.get('case_count', 0)} cases; {output.get('file_backed_case_count', 0)} file-backed",
|
||||
},
|
||||
{
|
||||
"label": "Runtime",
|
||||
"value": f"{conformance.get('pass_count', 0)}/{conformance.get('target_count', 0)}",
|
||||
"detail": "target conformance pass rate",
|
||||
},
|
||||
{
|
||||
"label": "Trust",
|
||||
"value": str(trust.get("secret_findings", 0)),
|
||||
"detail": f"{trust.get('script_count', 0)} scripts scanned; secrets found",
|
||||
},
|
||||
{
|
||||
"label": "Atlas",
|
||||
"value": str(atlas.get("route_collision_count", 0)),
|
||||
"detail": f"{atlas.get('skill_count', 0)} scanned skills; route collisions",
|
||||
},
|
||||
]
|
||||
return cards
|
||||
|
||||
|
||||
def render_gate_list(gates: list[dict[str, str]]) -> str:
|
||||
items = []
|
||||
for item in gates:
|
||||
link_html = f"<a href='{html.escape(item['link'])}'>证据</a>" if item.get("link") else ""
|
||||
items.append(
|
||||
"<article class='gate "
|
||||
+ html.escape(item["status"])
|
||||
+ "'>"
|
||||
f"<div><span>{html.escape(status_label(item['status']))}</span><h3>{html.escape(item['label'])}</h3></div>"
|
||||
f"<p>{html.escape(item['detail'])}</p>"
|
||||
f"<footer>{html.escape(item['evidence'])} {link_html}</footer>"
|
||||
"</article>"
|
||||
)
|
||||
return "".join(items)
|
||||
|
||||
|
||||
def render_insights(cards: list[dict[str, str]]) -> str:
|
||||
return "".join(
|
||||
(
|
||||
"<article class='metric'>"
|
||||
f"<span>{html.escape(item['label'])}</span>"
|
||||
f"<strong>{html.escape(item['value'])}</strong>"
|
||||
f"<p>{html.escape(item['detail'])}</p>"
|
||||
"</article>"
|
||||
)
|
||||
for item in cards
|
||||
)
|
||||
|
||||
|
||||
def render_issue_list(title: str, items: list[dict[str, str]]) -> str:
|
||||
if not items:
|
||||
return f"<section><h2>{html.escape(title)}</h2><p class='muted'>无。</p></section>"
|
||||
body = "".join(
|
||||
(
|
||||
"<li>"
|
||||
f"<strong>{html.escape(item['label'])}</strong>"
|
||||
f"<span>{html.escape(item['detail'])}</span>"
|
||||
"</li>"
|
||||
)
|
||||
for item in items
|
||||
)
|
||||
return f"<section><h2>{html.escape(title)}</h2><ul class='issues'>{body}</ul></section>"
|
||||
|
||||
|
||||
def render_html(report: dict[str, Any]) -> str:
|
||||
summary = report["summary"]
|
||||
gates = report["gates"]
|
||||
blockers = report["blockers"]
|
||||
warnings = report["warnings"]
|
||||
insights = insight_cards(report["data"])
|
||||
overview = report["data"]["overview"]
|
||||
manifest = report["data"]["manifest"]
|
||||
frontmatter = report["data"]["frontmatter"]
|
||||
title = overview.get("display_name") or overview.get("title") or frontmatter.get("name") or manifest.get("name") or "Skill"
|
||||
description = overview.get("description") or frontmatter.get("description", "")
|
||||
nav = [
|
||||
("#overview", "审查总览"),
|
||||
("#intent", "意图画布"),
|
||||
("#trigger", "触发实验"),
|
||||
("#output", "输出实验"),
|
||||
("#runtime", "运行矩阵"),
|
||||
("#trust", "信任报告"),
|
||||
("#atlas", "组合治理"),
|
||||
("#release", "发布路线"),
|
||||
]
|
||||
nav_html = "".join(f"<a href='{href}'>{label}</a>" for href, label in nav)
|
||||
gates_html = render_gate_list(gates)
|
||||
metrics_html = render_insights(insights)
|
||||
blockers_html = render_issue_list("阻断事项", blockers)
|
||||
warnings_html = render_issue_list("关注事项", warnings)
|
||||
output_summary = report["data"]["output_quality"].get("summary", {})
|
||||
conformance_summary = report["data"]["conformance"].get("summary", {})
|
||||
trust_summary = report["data"]["trust"].get("summary", {})
|
||||
atlas_summary = report["data"]["atlas"].get("summary", {})
|
||||
evidence_html = "".join(
|
||||
f"<li><strong>{html.escape(key)}</strong><span>{html.escape(value)}</span></li>"
|
||||
for key, value in report["evidence_paths"].items()
|
||||
)
|
||||
return f"""<!DOCTYPE html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>{html.escape(str(title))} Review Studio 2.0</title>
|
||||
<style>
|
||||
:root {{
|
||||
--ink: #1B365D;
|
||||
--text: #24201d;
|
||||
--muted: #746d66;
|
||||
--line: #e7ded2;
|
||||
--soft: #faf8f5;
|
||||
--pass: #1e6b52;
|
||||
--warn: #9a6718;
|
||||
--block: #9b2c2c;
|
||||
}}
|
||||
* {{ box-sizing: border-box; }}
|
||||
html {{ scroll-behavior: smooth; }}
|
||||
body {{
|
||||
margin: 0;
|
||||
background: #ffffff;
|
||||
color: var(--text);
|
||||
font-family: Georgia, "Times New Roman", "Songti SC", serif;
|
||||
line-height: 1.58;
|
||||
}}
|
||||
nav {{
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
display: flex;
|
||||
gap: 4px;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
padding: 10px 16px;
|
||||
background: rgba(255,255,255,0.94);
|
||||
border-bottom: 1px solid var(--line);
|
||||
backdrop-filter: blur(10px);
|
||||
}}
|
||||
nav a {{
|
||||
color: var(--ink);
|
||||
text-decoration: none;
|
||||
font-size: 14px;
|
||||
padding: 7px 10px;
|
||||
border-radius: 6px;
|
||||
}}
|
||||
nav a:hover {{ background: var(--soft); }}
|
||||
main {{ max-width: 1180px; margin: 0 auto; padding: 44px 28px 76px; }}
|
||||
header {{ border-bottom: 1px solid var(--line); padding-bottom: 28px; margin-bottom: 28px; }}
|
||||
.eyebrow {{ color: var(--ink); font-size: 14px; letter-spacing: .08em; text-transform: uppercase; }}
|
||||
h1, h2, h3 {{ color: var(--text); font-weight: 500; margin: 0; letter-spacing: 0; }}
|
||||
h1 {{ font-size: clamp(34px, 5vw, 64px); line-height: 1.03; max-width: 920px; margin-top: 12px; }}
|
||||
h2 {{ font-size: 30px; margin-bottom: 14px; }}
|
||||
h3 {{ font-size: 19px; }}
|
||||
p {{ margin: 0; }}
|
||||
.lede {{ max-width: 820px; color: var(--muted); font-size: 20px; margin-top: 18px; }}
|
||||
.decision {{
|
||||
display: inline-flex;
|
||||
align-items: baseline;
|
||||
gap: 12px;
|
||||
margin-top: 24px;
|
||||
padding: 12px 16px;
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
color: var(--ink);
|
||||
background: var(--soft);
|
||||
}}
|
||||
.decision strong {{ font-size: 28px; }}
|
||||
section {{ padding: 30px 0; border-bottom: 1px solid var(--line); scroll-margin-top: 76px; }}
|
||||
.metrics, .gates {{
|
||||
display: grid;
|
||||
grid-template-columns: repeat(5, minmax(0, 1fr));
|
||||
gap: 14px;
|
||||
}}
|
||||
.gates {{ grid-template-columns: repeat(4, minmax(0, 1fr)); }}
|
||||
.metric, .gate {{
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 16px;
|
||||
background: #fff;
|
||||
min-width: 0;
|
||||
}}
|
||||
.metric span, .gate span {{ display: block; color: var(--muted); font-size: 13px; }}
|
||||
.metric strong {{ display: block; color: var(--ink); font-size: 34px; line-height: 1.1; margin: 8px 0; }}
|
||||
.metric p, .gate p, .gate footer, .issues span, .evidence span {{ color: var(--muted); font-size: 14px; overflow-wrap: anywhere; }}
|
||||
.gate {{ display: flex; flex-direction: column; gap: 10px; }}
|
||||
.gate.pass {{ border-top: 4px solid var(--pass); }}
|
||||
.gate.warn {{ border-top: 4px solid var(--warn); }}
|
||||
.gate.block {{ border-top: 4px solid var(--block); }}
|
||||
.gate footer {{ border-top: 1px solid var(--line); padding-top: 10px; }}
|
||||
a {{ color: var(--ink); }}
|
||||
.twocol {{
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) minmax(0, 1fr);
|
||||
gap: 22px;
|
||||
align-items: start;
|
||||
}}
|
||||
.panel {{
|
||||
border: 1px solid var(--line);
|
||||
border-radius: 8px;
|
||||
padding: 18px;
|
||||
background: #fff;
|
||||
}}
|
||||
.panel p {{ color: var(--muted); }}
|
||||
.issues, .evidence {{
|
||||
list-style: none;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}}
|
||||
.issues li, .evidence li {{
|
||||
border-left: 3px solid var(--line);
|
||||
padding-left: 12px;
|
||||
display: grid;
|
||||
gap: 3px;
|
||||
}}
|
||||
.muted {{ color: var(--muted); }}
|
||||
@media (max-width: 980px) {{
|
||||
.metrics, .gates, .twocol {{ grid-template-columns: 1fr; }}
|
||||
main {{ padding: 32px 18px 60px; }}
|
||||
nav {{ justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; }}
|
||||
nav a {{ flex: 0 0 auto; }}
|
||||
}}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav>{nav_html}</nav>
|
||||
<main>
|
||||
<header id="overview">
|
||||
<div class="eyebrow">Review Studio 2.0</div>
|
||||
<h1>{html.escape(str(title))}</h1>
|
||||
<p class="lede">{html.escape(str(description))}</p>
|
||||
<div class="decision">
|
||||
<span>审查结论</span>
|
||||
<strong>{html.escape(summary['decision'])}</strong>
|
||||
<span>Score {html.escape(str(summary['world_class_score']))}/100</span>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<section>
|
||||
<h2>核心指标</h2>
|
||||
<div class="metrics">{metrics_html}</div>
|
||||
</section>
|
||||
|
||||
<section>
|
||||
<h2>审查闸门</h2>
|
||||
<div class="gates">{gates_html}</div>
|
||||
</section>
|
||||
|
||||
<div class="twocol">
|
||||
{blockers_html}
|
||||
{warnings_html}
|
||||
</div>
|
||||
|
||||
<section id="intent" class="twocol">
|
||||
<div class="panel">
|
||||
<h2>意图画布</h2>
|
||||
<p>{html.escape(str(report['data']['intent_confidence'].get('anchor_sentence', description)))}</p>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<h2>证据路径</h2>
|
||||
<ul class="evidence">{evidence_html}</ul>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section id="trigger" class="twocol">
|
||||
<div class="panel"><h2>触发实验</h2><p>{html.escape(gates[1]['detail'])}</p></div>
|
||||
<div class="panel"><h2>组合治理</h2><p>{html.escape(str(atlas_summary))}</p></div>
|
||||
</section>
|
||||
|
||||
<section id="output" class="twocol">
|
||||
<div class="panel"><h2>输出实验</h2><p>{html.escape(str(output_summary))}</p></div>
|
||||
<div class="panel"><h2>发布标准</h2><p>Governed 和 Library 至少需要 5 个 output eval cases,并覆盖 file-backed、near-neighbor 和 boundary case。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="runtime" class="twocol">
|
||||
<div class="panel"><h2>运行矩阵</h2><p>{html.escape(str(conformance_summary))}</p></div>
|
||||
<div class="panel"><h2>上下文</h2><p>{html.escape(gates[3]['detail'])}</p></div>
|
||||
</section>
|
||||
|
||||
<section id="trust" class="twocol">
|
||||
<div class="panel"><h2>信任报告</h2><p>{html.escape(str(trust_summary))}</p></div>
|
||||
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="atlas" class="twocol">
|
||||
<div class="panel"><h2>组合治理</h2><p>{html.escape(gates[6]['detail'])}</p></div>
|
||||
<div class="panel"><h2>下一动作</h2><p>优先处理真实 portfolio 中的 duplicate names、stale skills、owner gaps,再做 registry 或 telemetry。</p></div>
|
||||
</section>
|
||||
|
||||
<section id="release" class="twocol">
|
||||
<div class="panel"><h2>发布路线</h2><p>{html.escape(gates[7]['detail'])}</p></div>
|
||||
<div class="panel"><h2>世界级缺口</h2><p>下一阶段应继续推进 IR-first compiler、registry audit、telemetry drift loop,以及更严格的 governed trust gates。</p></div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def render_review_studio(skill_dir: Path, output_html: Path | None = None, output_json: Path | None = None) -> dict[str, Any]:
|
||||
skill_dir = skill_dir.resolve()
|
||||
reports_dir = skill_dir / "reports"
|
||||
reports_dir.mkdir(parents=True, exist_ok=True)
|
||||
output_html = output_html or reports_dir / "review-studio.html"
|
||||
output_json = output_json or reports_dir / "review-studio.json"
|
||||
data = load_review_data(skill_dir)
|
||||
gates = build_gates(skill_dir, output_html, data)
|
||||
blockers, warnings = add_blockers_from_gate(gates)
|
||||
score = weighted_score(gates)
|
||||
decision = "blocked" if blockers else ("review" if warnings else "ready")
|
||||
report = {
|
||||
"schema_version": "2.0",
|
||||
"ok": True,
|
||||
"skill_dir": display_path(skill_dir, skill_dir),
|
||||
"summary": {
|
||||
"decision": decision,
|
||||
"world_class_score": score,
|
||||
"gate_count": len(gates),
|
||||
"blocker_count": len(blockers),
|
||||
"warning_count": len(warnings),
|
||||
},
|
||||
"gates": gates,
|
||||
"blockers": blockers,
|
||||
"warnings": warnings,
|
||||
"evidence_paths": evidence_paths(skill_dir),
|
||||
"data": data,
|
||||
"artifacts": {
|
||||
"html": display_path(skill_dir, output_html),
|
||||
"json": display_path(skill_dir, output_json),
|
||||
},
|
||||
}
|
||||
output_html.write_text(render_html(report), encoding="utf-8")
|
||||
output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
return {key: value for key, value in report.items() if key != "data"}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Render Review Studio 2.0 for a skill package.")
|
||||
parser.add_argument("skill_dir", nargs="?", default=".")
|
||||
parser.add_argument("--output-html")
|
||||
parser.add_argument("--output-json")
|
||||
args = parser.parse_args()
|
||||
payload = render_review_studio(
|
||||
Path(args.skill_dir),
|
||||
output_html=Path(args.output_html).resolve() if args.output_html else None,
|
||||
output_json=Path(args.output_json).resolve() if args.output_json else None,
|
||||
)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -36,11 +36,24 @@ def normalize(text: str) -> str:
|
||||
return str(text).casefold()
|
||||
|
||||
|
||||
def validate_case(case: dict[str, Any]) -> list[str]:
|
||||
def validate_case(case: dict[str, Any], cases_root: Path) -> list[str]:
|
||||
failures = []
|
||||
for key in ("id", "prompt", "baseline_output", "with_skill_output", "assertions"):
|
||||
if key not in case:
|
||||
failures.append(f"{case.get('id', '<unknown>')}: missing {key}")
|
||||
for raw_path in case.get("input_files", []):
|
||||
rel = Path(str(raw_path))
|
||||
if rel.is_absolute():
|
||||
failures.append(f"{case.get('id', '<unknown>')}: input_files must be relative: {raw_path}")
|
||||
continue
|
||||
target = (cases_root / rel).resolve()
|
||||
try:
|
||||
target.relative_to(cases_root.resolve())
|
||||
except ValueError:
|
||||
failures.append(f"{case.get('id', '<unknown>')}: input_file escapes eval folder: {raw_path}")
|
||||
continue
|
||||
if not target.exists():
|
||||
failures.append(f"{case.get('id', '<unknown>')}: input_file is missing: {raw_path}")
|
||||
assertions = case.get("assertions", [])
|
||||
if not isinstance(assertions, list) or not assertions:
|
||||
failures.append(f"{case.get('id', '<unknown>')}: assertions must be a non-empty list")
|
||||
@@ -92,6 +105,8 @@ def grade_case(case: dict[str, Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"id": case["id"],
|
||||
"prompt": case["prompt"],
|
||||
"input_files": case.get("input_files", []),
|
||||
"metadata": case.get("metadata", {}),
|
||||
"baseline": baseline,
|
||||
"with_skill": with_skill,
|
||||
"delta": round(with_skill["score"] - baseline["score"], 2),
|
||||
@@ -106,8 +121,14 @@ def build_summary(results: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
with_skill_average = sum(item["with_skill"]["score"] for item in results) / case_count if case_count else 0
|
||||
regressions = [item for item in results if item["delta"] < 0]
|
||||
failures = sorted({failure for item in results for failure in item["failure_taxonomy"]})
|
||||
file_backed = [item for item in results if item.get("input_files")]
|
||||
near_neighbors = [item for item in results if item.get("metadata", {}).get("case_type") == "near_neighbor"]
|
||||
boundary_cases = [item for item in results if item.get("metadata", {}).get("case_type") == "boundary"]
|
||||
return {
|
||||
"case_count": case_count,
|
||||
"file_backed_case_count": len(file_backed),
|
||||
"near_neighbor_case_count": len(near_neighbors),
|
||||
"boundary_case_count": len(boundary_cases),
|
||||
"baseline_pass_rate": round(baseline_average, 2),
|
||||
"with_skill_pass_rate": round(with_skill_average, 2),
|
||||
"delta": round(with_skill_average - baseline_average, 2),
|
||||
@@ -162,7 +183,7 @@ def render_markdown(payload: dict[str, Any]) -> str:
|
||||
|
||||
def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dict[str, Any]:
|
||||
cases = load_cases(cases_path)
|
||||
validation_failures = [failure for case in cases for failure in validate_case(case)]
|
||||
validation_failures = [failure for case in cases for failure in validate_case(case, cases_path.parent)]
|
||||
if validation_failures:
|
||||
payload = {
|
||||
"ok": False,
|
||||
|
||||
@@ -202,6 +202,8 @@ def derive_strengths(skill_dir: Path, metadata: dict) -> list[str]:
|
||||
strengths.append("已生成 Security Trust Report,可审查脚本、依赖、secret 和包完整性风险。")
|
||||
if (skill_dir / "reports" / "skill_atlas.json").exists():
|
||||
strengths.append("已生成 Skill Atlas,可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。")
|
||||
if (skill_dir / "reports" / "review-studio.json").exists():
|
||||
strengths.append("已生成 Review Studio 2.0,可在一页中查看 blocker、warning、证据路径和发布闸门。")
|
||||
if (skill_dir / "agents" / "interface.yaml").exists():
|
||||
strengths.append("已打包 agents/interface.yaml,便于后续做跨平台适配。")
|
||||
if (skill_dir / "references").exists() and any((skill_dir / "references").iterdir()):
|
||||
@@ -500,6 +502,7 @@ def build_report_model(skill_dir: Path) -> dict:
|
||||
"reports/conformance_matrix.md",
|
||||
"reports/security_trust_report.md",
|
||||
"reports/skill_atlas.html",
|
||||
"reports/review-studio.html",
|
||||
"reports/skill-overview.html",
|
||||
],
|
||||
"flow": ["输入材料", "Skill 包体", "可复用能力"],
|
||||
|
||||
@@ -509,6 +509,7 @@ def command_quickstart(args: argparse.Namespace) -> int:
|
||||
next_steps = [
|
||||
"Open reports/skill-overview.html to review the generated Skill audit report.",
|
||||
"Open reports/intent-dialogue.md and tighten the real job, outputs, and exclusions.",
|
||||
"Open reports/review-studio.html to inspect the Review Studio 2.0 gate view before release.",
|
||||
"Open reports/review-viewer.html to explain the package to a first-time reviewer.",
|
||||
"Use reports/iteration-directions.md to choose only one high-value next move before adding more files.",
|
||||
]
|
||||
@@ -543,6 +544,7 @@ def command_quickstart(args: argparse.Namespace) -> int:
|
||||
"artifact_design_profile": payload.get("artifacts", {}).get("artifact_design_profile_md"),
|
||||
"prompt_quality_profile": payload.get("artifacts", {}).get("prompt_quality_profile_md"),
|
||||
"system_model": payload.get("artifacts", {}).get("system_model_md"),
|
||||
"review_studio": payload.get("artifacts", {}).get("review_studio_html"),
|
||||
"review_viewer": payload.get("artifacts", {}).get("review_viewer_html"),
|
||||
},
|
||||
},
|
||||
@@ -707,6 +709,18 @@ def command_review_viewer(args: argparse.Namespace) -> int:
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_review_studio(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir]
|
||||
if args.output_html:
|
||||
cmd.extend(["--output-html", args.output_html])
|
||||
if args.output_json:
|
||||
cmd.extend(["--output-json", args.output_json])
|
||||
result = run_script("render_review_studio.py", cmd)
|
||||
print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2))
|
||||
return 0 if result["ok"] else 2
|
||||
|
||||
|
||||
def command_reference_scan(args: argparse.Namespace) -> int:
|
||||
skill_dir = str(Path(args.skill_dir).resolve())
|
||||
cmd = [skill_dir]
|
||||
@@ -1196,6 +1210,12 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
review_viewer_cmd.add_argument("--output-json")
|
||||
review_viewer_cmd.set_defaults(func=command_review_viewer)
|
||||
|
||||
review_studio_cmd = subparsers.add_parser("review-studio", help="Render Review Studio 2.0 for a skill package.")
|
||||
review_studio_cmd.add_argument("skill_dir", nargs="?", default=".")
|
||||
review_studio_cmd.add_argument("--output-html")
|
||||
review_studio_cmd.add_argument("--output-json")
|
||||
review_studio_cmd.set_defaults(func=command_review_studio)
|
||||
|
||||
reference_scan_cmd = subparsers.add_parser(
|
||||
"reference-scan",
|
||||
help="Render a controlled benchmark scan report for a skill package.",
|
||||
|
||||
@@ -55,10 +55,9 @@
|
||||
"`Scaffold`: exploratory or personal.",
|
||||
"`Production`: team reuse.",
|
||||
"`Library`: shared infrastructure.",
|
||||
"a working skill directory",
|
||||
"a `SKILL.md`",
|
||||
"aligned `agents/interface.yaml`",
|
||||
"optional `references/`, `scripts/`, `evals/`, `reports/`, and `manifest.json` when justified"
|
||||
"Ask one focused clarification when the real job, output, or exclusion boundary is unclear.",
|
||||
"Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.",
|
||||
"Do not silently broaden the skill into adjacent jobs just because the examples are nearby."
|
||||
],
|
||||
"failure_modes": [
|
||||
"Tables can render as dense grids with weak hierarchy or poor mobile readability.",
|
||||
@@ -96,6 +95,7 @@
|
||||
"references/reference-scan.md",
|
||||
"references/regression-cause-taxonomy.md",
|
||||
"references/resource-boundaries.md",
|
||||
"references/review-studio-method.md",
|
||||
"references/runtime-conformance-method.md",
|
||||
"references/skill-archetypes.md",
|
||||
"references/skill-atlas-method.md",
|
||||
@@ -137,6 +137,7 @@
|
||||
"scripts/render_reference_scan.py",
|
||||
"scripts/render_reference_synthesis.py",
|
||||
"scripts/render_regression_history.py",
|
||||
"scripts/render_review_studio.py",
|
||||
"scripts/render_review_viewer.py",
|
||||
"scripts/render_skill_overview.py",
|
||||
"scripts/render_social_preview.py",
|
||||
@@ -213,6 +214,7 @@
|
||||
"evals/history/description_optimization/README.md",
|
||||
"evals/holdout/trigger_cases.json",
|
||||
"evals/output/cases.jsonl",
|
||||
"evals/output/fixtures/release-brief-source.md",
|
||||
"evals/output/schema.json",
|
||||
"evals/packaging_expectations.json",
|
||||
"evals/promotion_policy.md",
|
||||
@@ -222,6 +224,7 @@
|
||||
],
|
||||
"output": [
|
||||
"evals/output/cases.jsonl",
|
||||
"evals/output/fixtures/release-brief-source.md",
|
||||
"evals/output/schema.json"
|
||||
],
|
||||
"adversarial": [
|
||||
|
||||
@@ -51,6 +51,7 @@
|
||||
"scripts/render_reference_scan.py",
|
||||
"scripts/render_reference_synthesis.py",
|
||||
"scripts/render_regression_history.py",
|
||||
"scripts/render_review_studio.py",
|
||||
"scripts/render_review_viewer.py",
|
||||
"scripts/render_skill_overview.py",
|
||||
"scripts/render_social_preview.py",
|
||||
@@ -89,6 +90,7 @@
|
||||
"references/reference-scan.md",
|
||||
"references/regression-cause-taxonomy.md",
|
||||
"references/resource-boundaries.md",
|
||||
"references/review-studio-method.md",
|
||||
"references/runtime-conformance-method.md",
|
||||
"references/skill-archetypes.md",
|
||||
"references/skill-atlas-method.md",
|
||||
|
||||
@@ -10,6 +10,11 @@
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile"
|
||||
"portability_profile",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -10,6 +10,11 @@
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile"
|
||||
"portability_profile",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -10,6 +10,11 @@
|
||||
"trust_level",
|
||||
"remote_inline_execution",
|
||||
"degradation_strategy",
|
||||
"portability_profile"
|
||||
"portability_profile",
|
||||
"job_to_be_done",
|
||||
"ir_source",
|
||||
"ir_schema_version",
|
||||
"semantic_contract",
|
||||
"semantic_parity"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -53,6 +53,28 @@ def main() -> None:
|
||||
for field in snapshot.get("required_fields", []):
|
||||
if field not in adapter:
|
||||
failures.append(f"{name}: missing required adapter field {field}")
|
||||
if adapter.get("ir_source") != "skill-ir/examples/yao-meta-skill.json":
|
||||
failures.append(f"{name}: adapter is not sourced from root Skill IR")
|
||||
if adapter.get("ir_schema_version") != "2.0.0":
|
||||
failures.append(f"{name}: missing Skill IR schema version")
|
||||
contract = adapter.get("semantic_contract", {})
|
||||
if contract.get("name") != "yao-meta-skill":
|
||||
failures.append(f"{name}: semantic contract name mismatch")
|
||||
if contract.get("trigger_description") != adapter.get("description"):
|
||||
failures.append(f"{name}: trigger description is not adapter description")
|
||||
if contract.get("job_to_be_done") != adapter.get("job_to_be_done"):
|
||||
failures.append(f"{name}: job-to-be-done not carried into adapter")
|
||||
if contract.get("resource_counts", {}).get("references", 0) <= 0:
|
||||
failures.append(f"{name}: semantic contract does not include reference counts")
|
||||
if contract.get("eval_counts", {}).get("output", 0) <= 0:
|
||||
failures.append(f"{name}: semantic contract does not include output eval counts")
|
||||
parity = adapter.get("semantic_parity", {})
|
||||
if parity.get("source") != "skill-ir":
|
||||
failures.append(f"{name}: semantic parity source is not skill-ir")
|
||||
if parity.get("name_matches_ir") is not True:
|
||||
failures.append(f"{name}: frontmatter name does not match Skill IR")
|
||||
if parity.get("description_matches_ir") is not True:
|
||||
failures.append(f"{name}: frontmatter description does not match Skill IR")
|
||||
if not (TMP / snapshot["required_generated_file"]).exists():
|
||||
failures.append(f"{name}: missing generated file {snapshot['required_generated_file']}")
|
||||
if name == "openai":
|
||||
|
||||
@@ -32,7 +32,10 @@ def main() -> None:
|
||||
)
|
||||
payload = json.loads(proc.stdout)
|
||||
assert payload["ok"], payload
|
||||
assert payload["summary"]["case_count"] == 3, payload
|
||||
assert payload["summary"]["case_count"] == 5, payload
|
||||
assert payload["summary"]["file_backed_case_count"] == 1, payload
|
||||
assert payload["summary"]["near_neighbor_case_count"] == 1, payload
|
||||
assert payload["summary"]["boundary_case_count"] == 1, payload
|
||||
assert payload["summary"]["with_skill_pass_rate"] > payload["summary"]["baseline_pass_rate"], payload
|
||||
assert payload["summary"]["delta"] > 0, payload
|
||||
assert payload["summary"]["gate_pass"], payload
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
SCRIPT = ROOT / "scripts" / "render_review_studio.py"
|
||||
|
||||
|
||||
def main() -> None:
|
||||
tmp_root = ROOT / "tests" / "tmp_review_studio"
|
||||
tmp_root.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run([sys.executable, str(ROOT / "scripts" / "run_output_eval.py")], cwd=ROOT, check=True, capture_output=True, text=True)
|
||||
|
||||
output_html = tmp_root / "review-studio.html"
|
||||
output_json = tmp_root / "review-studio.json"
|
||||
proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
str(ROOT),
|
||||
"--output-html",
|
||||
str(output_html),
|
||||
"--output-json",
|
||||
str(output_json),
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
payload = json.loads(proc.stdout)
|
||||
assert payload["ok"], payload
|
||||
assert payload["schema_version"] == "2.0", payload
|
||||
assert payload["summary"]["gate_count"] == 8, payload
|
||||
assert payload["summary"]["world_class_score"] > 0, payload
|
||||
gate_keys = {item["key"] for item in payload["gates"]}
|
||||
assert {"intent-canvas", "trigger-lab", "output-lab", "runtime-matrix", "trust-report", "skill-atlas", "release-notes"} <= gate_keys, payload
|
||||
output_gate = next(item for item in payload["gates"] if item["key"] == "output-lab")
|
||||
assert output_gate["status"] == "pass", output_gate
|
||||
assert "5/5 cases" in output_gate["detail"], output_gate
|
||||
assert "file-backed 1" in output_gate["detail"], output_gate
|
||||
assert output_html.exists(), output_html
|
||||
assert output_json.exists(), output_json
|
||||
html = output_html.read_text(encoding="utf-8")
|
||||
assert "Review Studio 2.0" in html, html[:400]
|
||||
assert "审查闸门" in html, html[:1200]
|
||||
assert "输出实验" in html, html[:2000]
|
||||
assert str(ROOT) not in output_json.read_text(encoding="utf-8"), output_json
|
||||
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -83,6 +83,7 @@ def main() -> None:
|
||||
assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
|
||||
assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
|
||||
assert "reports/skill_atlas.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
|
||||
assert "reports/review-studio.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
|
||||
assert overview_json["skill_ir"]["schema_version"] in {"", "2.0.0"}, overview_json.get("skill_ir")
|
||||
assert "output_quality" in overview_json, overview_json.keys()
|
||||
assert "runtime_conformance" in overview_json, overview_json.keys()
|
||||
|
||||
@@ -45,6 +45,8 @@ def main() -> None:
|
||||
assert (created / "reports" / "intent-dialogue.md").exists(), created
|
||||
assert (created / "reports" / "intent-confidence.md").exists(), created
|
||||
assert (created / "reports" / "skill-overview.html").exists(), created
|
||||
assert (created / "reports" / "review-studio.html").exists(), created
|
||||
assert (created / "reports" / "review-studio.json").exists(), created
|
||||
assert (created / "reports" / "review-viewer.html").exists(), created
|
||||
assert (created / "reports" / "reference-scan.md").exists(), created
|
||||
assert (created / "reports" / "reference-synthesis.md").exists(), created
|
||||
@@ -58,7 +60,10 @@ def main() -> None:
|
||||
init_report_view = init_result["payload"]["report_view"]
|
||||
assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view
|
||||
assert Path(init_report_view["html_report"]).exists(), init_report_view
|
||||
assert init_report_view["review_studio"].endswith("reports/review-studio.html"), init_report_view
|
||||
assert Path(init_report_view["review_studio"]).exists(), init_report_view
|
||||
assert "Skill 已创建完成" in init_report_view["message"], init_report_view
|
||||
assert "Review Studio 2.0" in init_report_view["message"], init_report_view
|
||||
assert "概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view
|
||||
assert "默认使用中文简体" in init_report_view["message"], init_report_view
|
||||
assert "切换英文版" in init_report_view["message"], init_report_view
|
||||
@@ -90,6 +95,7 @@ def main() -> None:
|
||||
assert quickstart_result["ok"], quickstart_result
|
||||
quickstart_root = Path(quickstart_result["payload"]["root"])
|
||||
assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root
|
||||
assert (quickstart_root / "reports" / "review-studio.html").exists(), quickstart_root
|
||||
assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root
|
||||
assert (quickstart_root / "reports" / "intent-confidence.md").exists(), quickstart_root
|
||||
assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root
|
||||
@@ -105,9 +111,11 @@ def main() -> None:
|
||||
quickstart_report_view = quickstart_result["payload"]["report_view"]
|
||||
assert quickstart_report_view["html_report"].endswith("reports/skill-overview.html"), quickstart_report_view
|
||||
assert Path(quickstart_report_view["html_report"]).exists(), quickstart_report_view
|
||||
assert Path(quickstart_report_view["review_studio"]).exists(), quickstart_report_view
|
||||
assert "Skill 已创建完成" in quickstart_report_view["message"], quickstart_report_view
|
||||
assert "默认使用中文简体" in quickstart_report_view["message"], quickstart_report_view
|
||||
assert quickstart_result["payload"]["guidance"]["next_steps"][0].startswith("Open reports/skill-overview.html"), quickstart_result
|
||||
assert "reports/review-studio.html" in quickstart_result["payload"]["guidance"]["next_steps"][2], quickstart_result
|
||||
assert "audit report" in quickstart_result["payload"]["guidance"]["next_steps"][0], quickstart_result
|
||||
assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["reference_synthesis"].endswith(
|
||||
"reports/reference-synthesis.md"
|
||||
@@ -118,6 +126,9 @@ def main() -> None:
|
||||
assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["system_model"].endswith(
|
||||
"reports/system-model.md"
|
||||
), quickstart_result
|
||||
assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["review_studio"].endswith(
|
||||
"reports/review-studio.html"
|
||||
), quickstart_result
|
||||
assert "uncertainty_or_conflict" not in quickstart_result["payload"], quickstart_result
|
||||
quickstart_manifest = json.loads((quickstart_root / "manifest.json").read_text(encoding="utf-8"))
|
||||
assert quickstart_manifest["status"] == "active", quickstart_manifest
|
||||
@@ -162,6 +173,11 @@ def main() -> None:
|
||||
assert review_viewer_result["ok"], review_viewer_result
|
||||
assert review_viewer_result["payload"]["artifacts"]["html"].endswith("reports/review-viewer.html"), review_viewer_result
|
||||
|
||||
review_studio_result = run("review-studio", str(created))
|
||||
assert review_studio_result["ok"], review_studio_result
|
||||
assert review_studio_result["payload"]["artifacts"]["html"].endswith("reports/review-studio.html"), review_studio_result
|
||||
assert review_studio_result["payload"]["summary"]["gate_count"] == 8, review_studio_result
|
||||
|
||||
reference_scan_result = run(
|
||||
"reference-scan",
|
||||
str(created),
|
||||
|
||||
Reference in New Issue
Block a user