diff --git a/Makefile b/Makefile index 423121af..dd1cdc63 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -40,6 +40,21 @@ skill-report-metrics-check: skill-report-charts-check: $(PYTHON) tests/verify_skill_report_charts.py +skill-ir-check: + $(PYTHON) tests/verify_skill_ir.py + +output-eval-check: + $(PYTHON) tests/verify_output_eval_lab.py + +runtime-conformance-check: + $(PYTHON) tests/verify_conformance_suite.py + +trust-check: + $(PYTHON) tests/verify_trust_check.py + +skill-atlas-check: + $(PYTHON) tests/verify_skill_atlas.py + review-viewer-check: $(PYTHON) tests/verify_review_viewer.py @@ -136,10 +151,11 @@ sync-local-install: sync-active-install: $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check output-eval-check runtime-conformance-check trust-check skill-atlas-check review-viewer-check feedback-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_output_eval tests/tmp_conformance tests/tmp_trust tests/tmp_skill_atlas tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_feedback tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 0108dcc3..f1d66c68 100644 --- a/README.md +++ b/README.md @@ -103,6 +103,11 @@ python3 scripts/yao.py feedback my-skill --note "Tighten exclusions before addin python3 scripts/yao.py baseline-compare python3 scripts/yao.py check-update python3 scripts/yao.py package . --platform generic --output-dir dist +python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json +python3 scripts/yao.py output-eval +python3 scripts/yao.py conformance . +python3 scripts/yao.py trust . +python3 scripts/yao.py skill-atlas --workspace-root . ``` ## Local Development Source @@ -357,6 +362,11 @@ Utility scripts that make the meta-skill operational: - `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage - `render_skill_overview.py`: generates the white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, top-right language switch, v2 scorecard, inline SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap +- `export_skill_ir.py`: exports the 2.0 platform-neutral Skill IR contract from `SKILL.md`, manifest, interface metadata, evals, resources, and reports +- `run_output_eval.py`: runs the Output Eval Lab v0 with static with-skill vs baseline assertion grading and writes `reports/output_quality_scorecard.*` +- `run_conformance_suite.py`: verifies runtime conformance for OpenAI, Claude, Agent Skills, VS Code/Copilot-style, and generic targets +- `trust_check.py`: generates the trust/security report for scripts, dependencies, secret risk, trust metadata, and package integrity +- `build_skill_atlas.py`: builds the Skill Atlas catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview for a multi-skill workspace - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain - `check_update.py`: checks GitHub for a newer `VERSION` or remote manifest version and reports a reinstall hint without modifying local files - `render_output_risk_profile.py`: predicts output-specific failure modes such as generic headings, citation clutter, screenshot mistakes, weak Markdown tables, and missing execution assumptions diff --git a/SKILL.md b/SKILL.md index 379e0921..e15f63cc 100644 --- a/SKILL.md +++ b/SKILL.md @@ -32,7 +32,7 @@ Mode rules: [Method](references/skill-engineering-method.md), [Operating Modes]( 6. Add only folders and gates that earn their keep. 7. Surface the top three next iteration directions. -Core playbooks: [Method](references/skill-engineering-method.md), [Intent Dialogue](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Artifact Design](references/artifact-design-doctrine.md), [Prompt Engineering](references/prompt-engineering-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md). +Core playbooks: [Method](references/skill-engineering-method.md), [Intent Dialogue](references/intent-dialogue.md), [Reference Scan](references/reference-scan.md), [Skill IR](references/skill-ir-method.md), [Output Eval](references/output-eval-method.md), [Artifact Design](references/artifact-design-doctrine.md), [Prompt Engineering](references/prompt-engineering-doctrine.md), [Systems Thinking](references/systems-thinking-doctrine.md). ## First-Turn Style diff --git a/docs/migration-v2.md b/docs/migration-v2.md new file mode 100644 index 00000000..0b4a349b --- /dev/null +++ b/docs/migration-v2.md @@ -0,0 +1,61 @@ +# Migration To Skill OS 2.0 + +This document tracks the first migration path from Yao Meta Skill 1.x to the Skill OS 2.0 architecture. + +## 1.x To 2.0 Mapping + +| 1.x Asset | 2.0 Role | +| --- | --- | +| `SKILL.md` frontmatter | `skill-ir.trigger_surface.description` | +| `reports/intent-context.json` | `skill-ir.job_to_be_done` and input/output assumptions | +| `evals/trigger_cases.json` | `skill-ir.trigger_surface` samples and Trigger Eval Lab | +| `reports/baseline-compare.*` | legacy baseline evidence for Output Eval Lab | +| `reports/output-risk-profile.*` | failure taxonomy seed for Output Eval Lab | +| `agents/interface.yaml` | target compatibility seed for runtime conformance | +| `manifest.json` | governance and registry metadata seed | + +## Migration Steps + +1. Export Skill IR: + + ```bash + python3 scripts/export_skill_ir.py . --output-json skill-ir/examples/yao-meta-skill.json + ``` + +2. Run Output Eval Lab: + + ```bash + python3 scripts/run_output_eval.py + ``` + +3. Run runtime conformance and trust checks: + + ```bash + python3 scripts/run_conformance_suite.py . + python3 scripts/trust_check.py . + ``` + +4. For library or team workspaces, build the Skill Atlas: + + ```bash + python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json + ``` + +5. Keep legacy trigger eval gates intact while output eval coverage grows. + +6. Move adapter-specific logic toward future compiler commands after IR fields are stable. + +7. Treat new 2.0 checks as tiered gates: scaffold stays light, production adds output eval, library adds conformance and atlas, governed adds trust and release evidence. + +## Compatibility Strategy + +No existing 1.x command is removed in the v0 migration. New commands produce additional artifacts and should become release gates only after their cases and schemas stabilize. + +## Deprecation Notes + +- Legacy baseline compare remains useful but should not be treated as full output quality proof. +- Platform packaging should continue to work directly until compiler-based packaging is introduced. +- Review Studio 2.0 should aggregate existing reports before replacing any current report. +- Conformance v0 is a structural and metadata gate; it does not yet simulate each client runtime. +- Trust v0 blocks obvious secret and remote-inline-execution risks, while surfacing script and dependency warnings for reviewer judgment. +- Skill Atlas v0 is local-first portfolio analysis; avoid committing reports generated from private parent workspaces unless the names and routes are intended to be shared. diff --git a/evals/README.md b/evals/README.md index 0a63885c..f36f226c 100644 --- a/evals/README.md +++ b/evals/README.md @@ -16,6 +16,7 @@ Contents: - `../reports/description_optimization*.{json,md}`: generated route-optimization reports for the root skill - `failure-cases.md`: current weak spots and regression targets - `packaging_expectations.json`: required packaging behaviors for supported targets +- `output/`: Output Eval Lab cases and schema for with-skill vs baseline assertion grading - `../reports/`: generated suite JSON plus the homepage-visible family summary panel source Use: @@ -35,6 +36,8 @@ python3 tests/verify_route_confusion.py python3 tests/verify_failure_regressions.py python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip python3 tests/verify_packager_failures.py +python3 scripts/run_output_eval.py +python3 tests/verify_output_eval_lab.py ``` Regression scope now includes: @@ -54,3 +57,4 @@ Regression scope now includes: - judge-backed blind-holdout verification that adds a rubric-based second opinion for blind prompts - calibration summaries that surface score gaps, threshold margins, and risk bands for each acceptance gate - family-level drift history that records which blind and adversarial families stay clean over time +- output-quality scorecards that compare without-skill and with-skill artifacts through assertion grading diff --git a/evals/output/cases.jsonl b/evals/output/cases.jsonl new file mode 100644 index 00000000..70d1b6db --- /dev/null +++ b/evals/output/cases.jsonl @@ -0,0 +1,3 @@ +{"id":"skill-package-contract","prompt":"Turn this repeated workflow into a reusable team skill package.","baseline_output":"I can write a prompt for that workflow and include a short checklist.","with_skill_output":"Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/.","assertions":[{"id":"has-entrypoint","description":"Output names the routeable Skill entrypoint.","required":["SKILL.md"],"weight":1,"failure_type":"missing_entrypoint"},{"id":"has-interface","description":"Output includes neutral interface metadata.","required":["agents/interface.yaml"],"weight":1,"failure_type":"missing_interface"},{"id":"has-report","description":"Output points reviewer to generated overview report.","required":["reports/skill-overview.html"],"weight":1,"failure_type":"missing_review_artifact"},{"id":"has-resource-boundary","description":"Output preserves entrypoint/resource separation.","required":["references/","scripts/","reports/"],"weight":1,"failure_type":"resource_boundary"}],"human_review":{"expected_winner":"with_skill"}} +{"id":"output-eval-expectation","prompt":"Upgrade this production skill so we know whether the generated output is better than baseline.","baseline_output":"Add more examples and run the trigger eval again.","with_skill_output":"Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md.","assertions":[{"id":"has-baseline-comparison","description":"Output explicitly compares with-skill and baseline outputs.","required":["with-skill","baseline"],"weight":1,"failure_type":"missing_baseline_delta"},{"id":"has-assertion-grading","description":"Output includes objective assertion grading.","required":["assertion grading"],"weight":1,"failure_type":"missing_assertions"},{"id":"has-scorecard","description":"Output produces a scorecard report path.","required":["reports/output_quality_scorecard.md"],"weight":1,"failure_type":"missing_scorecard"},{"id":"has-failure-taxonomy","description":"Output records failure taxonomy.","required":["failure taxonomy"],"weight":1,"failure_type":"missing_failure_taxonomy"}],"human_review":{"expected_winner":"with_skill"}} +{"id":"ir-before-packaging","prompt":"Package this skill for OpenAI, Claude, Agent Skills, and generic targets.","baseline_output":"Export folders for each platform and adjust files as needed.","with_skill_output":"Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract.","assertions":[{"id":"has-ir-first","description":"Output requires Skill IR before packaging.","required":["Skill IR first"],"weight":1,"failure_type":"missing_ir"},{"id":"has-core-ir-fields","description":"Output lists core IR contract fields.","required":["job_to_be_done","trigger_surface","eval_plan","governance"],"weight":1,"failure_type":"incomplete_ir_contract"},{"id":"has-targets","description":"Output names requested runtime targets.","required":["OpenAI","Claude","Agent Skills","generic"],"weight":1,"failure_type":"missing_targets"},{"id":"has-semantic-preservation","description":"Output says targets must preserve the capability contract.","required":["preserve the same capability contract"],"weight":1,"failure_type":"semantic_drift"}],"human_review":{"expected_winner":"with_skill"}} diff --git a/evals/output/schema.json b/evals/output/schema.json new file mode 100644 index 00000000..675e24a7 --- /dev/null +++ b/evals/output/schema.json @@ -0,0 +1,65 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.example/output-eval-case.schema.json", + "title": "Yao Output Eval Case", + "description": "Static v0 output-eval case for comparing without-skill and with-skill outputs.", + "type": "object", + "additionalProperties": false, + "required": ["id", "prompt", "baseline_output", "with_skill_output", "assertions"], + "properties": { + "id": { + "type": "string", + "minLength": 1 + }, + "prompt": { + "type": "string", + "minLength": 1 + }, + "input_files": { + "type": "array", + "items": {"type": "string"} + }, + "baseline_output": { + "type": "string" + }, + "with_skill_output": { + "type": "string" + }, + "assertions": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id", "description"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "description": {"type": "string", "minLength": 1}, + "required": { + "type": "array", + "items": {"type": "string"} + }, + "forbidden": { + "type": "array", + "items": {"type": "string"} + }, + "weight": { + "type": "number", + "minimum": 0 + }, + "failure_type": { + "type": "string" + } + } + } + }, + "human_review": { + "type": "object", + "additionalProperties": true + }, + "metadata": { + "type": "object", + "additionalProperties": true + } + } +} diff --git a/references/output-eval-method.md b/references/output-eval-method.md new file mode 100644 index 00000000..02947a92 --- /dev/null +++ b/references/output-eval-method.md @@ -0,0 +1,47 @@ +# Output Eval Method + +Output Eval Lab proves whether a skill improves the final user-facing result, not only whether it routes correctly. + +## When To Use + +Use output evals for production, library, governed, or team-distributed skills. Scaffold skills can start with one smoke case, but production and above should show a positive with-skill vs baseline signal before promotion. + +## Case Design + +Each case should include: + +- a real prompt or task shape +- any required input files +- a baseline output that represents doing the task without the skill +- a with-skill output that represents the skill-guided behavior +- assertions that can be checked without subjective guessing +- optional human review notes for taste, completeness, or judgment + +## Assertion Rules + +Prefer assertions that catch material quality: + +- required deliverable paths +- required sections or contracts +- required boundary or exclusion language +- required evidence paths +- forbidden generic placeholders +- forbidden unsafe actions + +Avoid assertions that only reward wording memorization. If a case can pass by parroting one phrase while failing the real job, the assertion is too narrow. + +## Score Reading + +The first v0 scorecard reports: + +- baseline pass rate +- with-skill pass rate +- absolute delta +- failed assertions and failure taxonomy +- recommended next fixes + +Production promotion should require the with-skill pass rate to beat baseline and should explain every failed assertion. + +## Anti-Overfitting + +Keep a small public smoke set and a separate holdout set. Rotate real failures into the taxonomy instead of editing only the prompt that failed. Add near-neighbor cases whenever the output looks good but the boundary is still unclear. diff --git a/references/runtime-conformance-method.md b/references/runtime-conformance-method.md new file mode 100644 index 00000000..8bef7fde --- /dev/null +++ b/references/runtime-conformance-method.md @@ -0,0 +1,21 @@ +# Runtime Conformance Method + +Runtime conformance turns platform compatibility from a packaging afterthought into a release gate. + +## Purpose + +Use this check when a skill is packaged for OpenAI, Claude, Agent Skills, VS Code / Copilot, or generic targets. The goal is not to prove that every runtime behaves identically. The goal is to prove that the package exposes enough metadata, files, and degradation notes for each runtime to consume it safely. + +## V0 Checks + +- `SKILL.md` exists and has frontmatter `name` and `description`. +- `description` stays within the 1024 character limit used by common Agent Skills clients. +- `manifest.json` includes name, version, owner, maturity, status, review cadence, and target platforms. +- `agents/interface.yaml` includes display text, default prompt, activation mode, execution context, trust metadata, adapter targets, and degradation strategy. +- Skill IR exists and matches the frontmatter name and description. +- Resources named by Skill IR are relative paths and resolve inside the package. +- Unsupported or lossy target behavior is represented by a degradation note. + +## Reviewer Gate + +A reviewer should be able to see a target matrix with pass/fail status, failures, warnings, and artifact paths. Any failed target blocks library, governed, or team-distributed release for that target. diff --git a/references/skill-atlas-method.md b/references/skill-atlas-method.md new file mode 100644 index 00000000..4bcb4738 --- /dev/null +++ b/references/skill-atlas-method.md @@ -0,0 +1,22 @@ +# Skill Atlas Method + +Skill Atlas is the 2.0 operating layer for a workspace that contains many skills. + +## Purpose + +Single-skill quality is not enough for a team library. A skill portfolio also needs to reveal route collisions, stale ownership, duplicate resources, and repeated no-route opportunities. + +## V0 Checks + +- Catalog every `SKILL.md` under a workspace. +- Extract name, description, owner, maturity, targets, updated date, and review cadence. +- Detect similar descriptions as route-overlap candidates. +- Detect duplicate skill names. +- Detect shared script/reference filenames as dependency signals. +- Flag missing owner or review metadata. +- Flag stale skills based on `updated_at` and `review_cadence`. +- Extract no-route opportunities from failure notes. + +## Reviewer Gate + +Use Atlas before promoting a single skill into a shared library. If a route collision or missing owner appears, fix the portfolio boundary before adding more local complexity to one skill. diff --git a/references/skill-ir-method.md b/references/skill-ir-method.md new file mode 100644 index 00000000..574c83e9 --- /dev/null +++ b/references/skill-ir-method.md @@ -0,0 +1,41 @@ +# Skill IR Method + +Skill IR is the 2.0 layer that separates durable skill meaning from platform packaging. + +## Purpose + +Use Skill IR before platform-specific packaging for production, library, governed, or team-distributed skills. The IR should preserve the capability contract even when OpenAI, Claude, Agent Skills, VS Code, or generic adapters differ in folder layout, metadata names, or activation behavior. + +## What Belongs In IR + +- the recurring job the skill owns +- the frontmatter trigger description +- should-trigger, should-not-trigger, and near-neighbor edge cases +- workflow steps, decision points, and failure modes +- references, scripts, assets, and reports used by the package +- trigger and output eval plans +- output risk, execution risk, and trust boundary +- owner, maturity, review cadence, and target platforms + +## What Does Not Belong In IR + +- platform-specific file paths that only exist after packaging +- copied prose from external benchmarks +- client-specific UI labels +- local private paths that are not part of the skill package +- speculative adapters that are not requested or tested + +## Authoring Rule + +Export or update Skill IR before adding new adapters, compilers, registries, or conformance checks. If a field cannot be derived from local package evidence, leave it empty or use an explicit low-confidence default instead of inventing detail. + +## Reviewer Gate + +A reviewer should be able to answer: + +1. What does this skill own? +2. When should it trigger? +3. When should it not trigger? +4. Which resources and scripts carry the real behavior? +5. Which evals prove the contract? +6. Which targets can consume the skill without semantic loss? diff --git a/references/trust-security-method.md b/references/trust-security-method.md new file mode 100644 index 00000000..444820d9 --- /dev/null +++ b/references/trust-security-method.md @@ -0,0 +1,26 @@ +# Trust Security Method + +Trust checks make skills safer to install and review, especially when they include scripts or are distributed to a team. + +## When To Run + +Run the trust report when: + +- the skill contains scripts +- the skill will be shared with a team +- the package may be installed from a registry or plugin +- the skill reads external files, uses network access, or shells out +- the maturity tier is library or governed + +## V0 Checks + +- obvious secret patterns +- script help surface and interactive prompts +- dependency pinning +- runtime trust metadata +- network-capable scripts +- package integrity digest + +## Release Rule + +High-risk secrets or unrestricted remote inline execution block governed release. Warnings are reviewer-visible but do not block v0 unless the release owner decides the target environment requires stricter policy. diff --git a/reports/artifact-design-profile.json b/reports/artifact-design-profile.json index 61e0c9ee..7bada95b 100644 --- a/reports/artifact-design-profile.json +++ b/reports/artifact-design-profile.json @@ -1,16 +1,28 @@ { "skill_name": "yao-meta-skill", - "design_system": "content-led editorial", + "design_system": "evidence frame", "primary_artifact": { - "key": "report", - "label": "Report or brief", - "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "key": "code_or_cli", + "label": "Code, CLI, or implementation guide", + "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", "matched_keywords": [ - "report", - "summary" + "code", + "script", + "command" ] }, "artifact_families": [ + { + "key": "code_or_cli", + "label": "Code, CLI, or implementation guide", + "score": 3, + "matched_keywords": [ + "code", + "script", + "command" + ], + "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit." + }, { "key": "report", "label": "Report or brief", @@ -22,24 +34,24 @@ "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." }, { - "key": "code_or_cli", - "label": "Code, CLI, or implementation guide", + "key": "visual_capture", + "label": "Screenshot or visual evidence", "score": 1, "matched_keywords": [ - "script" + "screenshot" ], - "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit." + "direction": "Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect." } ], "layout_patterns": [ - "thesis", - "evidence blocks", - "decision table", - "risks", - "next actions", "prerequisites", "commands", - "expected output" + "expected output", + "failure handling", + "rollback or cleanup", + "thesis", + "evidence blocks", + "decision table" ], "design_tokens": { "type": [ @@ -64,12 +76,14 @@ ] }, "quality_gates": [ + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding.", "Keep the first screen useful without requiring the reader to parse every detail.", "Use tables only for comparisons; move explanations below the table.", "Keep source notes readable without flooding the body with markers.", - "Name the working directory and required inputs for commands.", - "Mark destructive, networked, or external side-effect operations.", - "Prefer the smallest runnable snippet over broad framework scaffolding." + "Never invent missing screenshots or visual states.", + "Record source, viewport, and crop intent." ], "anti_patterns": [ "Do not copy Kami's fixed parchment background as a default.", diff --git a/reports/artifact-design-profile.md b/reports/artifact-design-profile.md index 6d69e0b0..ed5736ee 100644 --- a/reports/artifact-design-profile.md +++ b/reports/artifact-design-profile.md @@ -1,36 +1,41 @@ # Artifact Design Profile Skill: `yao-meta-skill` -Design system: `content-led editorial` +Design system: `evidence frame` ## Primary Artifact Direction -**Report or brief** +**Code, CLI, or implementation guide** -High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail. +Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit. ## Matched Artifact Families +### Code, CLI, or implementation guide +- Matched keywords: code, script, command +- Score: `3` +- Direction: Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit. + ### Report or brief - Matched keywords: report, summary - Score: `2` - Direction: High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail. -### Code, CLI, or implementation guide -- Matched keywords: script +### Screenshot or visual evidence +- Matched keywords: screenshot - Score: `1` -- Direction: Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit. +- Direction: Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect. ## Layout Patterns To Prefer -- thesis -- evidence blocks -- decision table -- risks -- next actions - prerequisites - commands - expected output +- failure handling +- rollback or cleanup +- thesis +- evidence blocks +- decision table ## Design Tokens @@ -56,12 +61,14 @@ High-trust editorial report with a clear first-screen thesis, compact evidence b ## Quality Gates -- Keep the first screen useful without requiring the reader to parse every detail. -- Use tables only for comparisons; move explanations below the table. -- Keep source notes readable without flooding the body with markers. - Name the working directory and required inputs for commands. - Mark destructive, networked, or external side-effect operations. - Prefer the smallest runnable snippet over broad framework scaffolding. +- Keep the first screen useful without requiring the reader to parse every detail. +- Use tables only for comparisons; move explanations below the table. +- Keep source notes readable without flooding the body with markers. +- Never invent missing screenshots or visual states. +- Record source, viewport, and crop intent. ## Anti-Patterns diff --git a/reports/benchmark_methodology.md b/reports/benchmark_methodology.md new file mode 100644 index 00000000..91fa5ff0 --- /dev/null +++ b/reports/benchmark_methodology.md @@ -0,0 +1,66 @@ +# Benchmark Methodology + +This report makes Yao Meta Skill benchmark claims auditable. It distinguishes self-eval evidence from release gates and names the limits of every comparison. + +## Benchmark Types + +| Type | Purpose | Evidence | +| --- | --- | --- | +| Self-eval | Fast local regression signal during authoring | Trigger suites, output eval cases, packaging checks | +| Internal blind eval | Guard against tuning only visible cases | Blind holdout, adversarial holdout, route confusion | +| External review | Check whether the workflow holds up outside the author context | Human review notes, benchmark scans, release snapshots | + +## Sample Sources + +- Public fixtures that can be committed and rerun. +- Real but anonymized cases when user content can be safely reduced to metadata or synthetic prompts. +- Failure library entries from prior regressions. +- Near-neighbor prompts that should not trigger the skill. +- Output cases with without-skill and with-skill artifacts. + +## Evaluation Dimensions + +| Dimension | Evidence | +| --- | --- | +| Trigger reliability | should-trigger, should-not-trigger, near-neighbor, route confusion | +| Output effectiveness | with-skill vs baseline delta, v1 vs v2 delta, assertion grading | +| Context efficiency | entrypoint size, references split, resource-boundary score | +| Runtime compatibility | target package structure, metadata, degradation notes | +| Trust and security | script interface, dependency pinning, permissions, secret scan, package hash | +| Governance and drift | owner, maturity, review cadence, regression history, promotion decisions | +| UX and adoption | quickstart, review viewer, report readability, reviewer handoff | + +## Weighting Rule + +Any weighted public score must publish: + +- exact sample count +- case families +- weights per dimension +- commands used to reproduce results +- failing cases and excluded cases +- commit hash and generated artifact paths + +If a score is based on local self-eval only, label it as project-level self-eval rather than external benchmark proof. + +## Failure Disclosure + +Every release should keep at least one representative failure when failures exist. The release note should say: + +- what failed +- why it mattered +- whether the fix was trigger, output, runtime, trust, or governance related +- which test now prevents recurrence + +## Reproduction + +Recommended release evidence: + +```bash +git rev-parse HEAD +python3 scripts/run_output_eval.py +python3 scripts/export_skill_ir.py . --output-json skill-ir/examples/yao-meta-skill.json +make ci-test +``` + +Record generated artifacts in `reports/` and avoid comparing different runtime targets as if their capabilities were identical. diff --git a/reports/conformance_matrix.json b/reports/conformance_matrix.json new file mode 100644 index 00000000..b51addc7 --- /dev/null +++ b/reports/conformance_matrix.json @@ -0,0 +1,239 @@ +{ + "ok": true, + "skill": "yao-meta-skill", + "targets": [ + { + "target": "openai", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares openai", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "adapter target declares openai", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "openai degradation note exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [] + }, + { + "target": "claude", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares claude", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "adapter target declares claude", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "claude degradation note exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [] + }, + { + "target": "agent-skills", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "directory name matches skill name", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares agent-skills via agent-skills-compatible", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "canonical format is agent-skills", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [ + "agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + ] + }, + { + "target": "vscode", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "directory name matches skill name", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares vscode via agent-skills-compatible", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "canonical format is agent-skills", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [ + "vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0." + ] + }, + { + "target": "generic", + "status": "pass", + "checks": [ + "SKILL.md exists", + "frontmatter name exists", + "frontmatter description exists", + "description length <= 1024", + "name is runtime-safe", + "manifest.name exists", + "manifest.version exists", + "manifest.owner exists", + "manifest.status exists", + "manifest.maturity_tier exists", + "manifest.review_cadence exists", + "manifest declares generic", + "interface.display_name exists", + "interface.short_description exists", + "interface.default_prompt exists", + "adapter target declares generic", + "activation mode exists", + "execution context exists", + "execution shell exists", + "trust source tier exists", + "generic degradation note exists", + "Skill IR schema_version is 2.0.0", + "Skill IR name matches frontmatter", + "Skill IR description matches frontmatter", + "references resource resolves: references/artifact-design-doctrine.md", + "references resource resolves: references/authoring-discipline.md", + "references resource resolves: references/eval-playbook.md", + "references resource resolves: references/gate-selection.md", + "references resource resolves: references/governance.md", + "references resource resolves: references/human-review-template.md", + "references resource resolves: references/intent-dialogue.md", + "references resource resolves: references/iteration-philosophy.md", + "references resource resolves: references/non-skill-decision-tree.md", + "references resource resolves: references/operating-modes.md", + "references resource resolves: references/output-eval-method.md", + "references resource resolves: references/output-quality-risk.md" + ], + "failures": [], + "warnings": [] + } + ], + "summary": { + "target_count": 5, + "pass_count": 5, + "fail_count": 0 + }, + "artifacts": { + "json": "reports/conformance_matrix.json", + "markdown": "reports/conformance_matrix.md" + } +} diff --git a/reports/conformance_matrix.md b/reports/conformance_matrix.md new file mode 100644 index 00000000..d62a0847 --- /dev/null +++ b/reports/conformance_matrix.md @@ -0,0 +1,19 @@ +# Runtime Conformance Matrix + +- Skill: `yao-meta-skill` +- Targets: `5` +- Passed: `5` +- Failed: `0` + +| Target | Status | Failures | Warnings | +| --- | --- | --- | --- | +| openai | pass | None | None | +| claude | pass | None | None | +| agent-skills | pass | None | agent-skills uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0. | +| vscode | pass | None | vscode uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0. | +| generic | pass | None | None | + +## Reviewer Notes + +- Failed targets block release for that target. +- Warnings identify lossy or not-yet-compiled behavior that must remain visible. diff --git a/reports/context_budget.json b/reports/context_budget.json index c99f1788..d5794bd7 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -5,13 +5,13 @@ "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, - "skill_body_tokens": 787, - "other_text_tokens": 375451, - "estimated_initial_load_tokens": 963, - "estimated_total_text_tokens": 376238, - "relevant_file_count": 213, + "skill_body_tokens": 810, + "other_text_tokens": 548453, + "estimated_initial_load_tokens": 986, + "estimated_total_text_tokens": 549263, + "relevant_file_count": 251, "unused_resource_dirs": [], "quality_signal_points": 130, - "quality_density": 135.0 + "quality_density": 131.8 } } diff --git a/reports/context_budget.md b/reports/context_budget.md index 95b52d14..5b08c50c 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | -| root | `.` | `production` | 1000 | 963 | 787 | 135.0 | - | ok | +| root | `.` | `production` | 1000 | 986 | 810 | 131.8 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 230f2917..5cf4efb6 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -6,9 +6,9 @@ "path": ".", "budget_tier": "production", "budget_limit": 1000, - "initial_tokens": 963, - "skill_body_tokens": 787, - "quality_density": 135.0, + "initial_tokens": 986, + "skill_body_tokens": 810, + "quality_density": 131.8, "unused_resource_dirs": [], "ok": true, "warnings": [] diff --git a/reports/github-benchmark-scan.json b/reports/github-benchmark-scan.json new file mode 100644 index 00000000..a76b0598 --- /dev/null +++ b/reports/github-benchmark-scan.json @@ -0,0 +1,188 @@ +{ + "ok": true, + "query": "skills", + "source": "github-api", + "network_boundary": "github-api-only", + "repositories": [ + { + "full_name": "obra/superpowers", + "html_url": "https://github.com/obra/superpowers", + "description": "An agentic skills framework & software development methodology that works.", + "stars": 226125, + "topics": [], + "primary_language": "Shell", + "patterns": [ + { + "category": "method", + "matched": "process", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale." + }, + { + "category": "execution", + "matched": "cli", + "borrow": "Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not copy repo-specific commands or environment assumptions verbatim." + }, + { + "category": "evaluation", + "matched": "test", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here." + }, + { + "category": "structure", + "matched": "docs", + "borrow": "Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not mirror documentation bulk that adds context cost without improving reliability." + } + ], + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure.", + "Borrow explicit validation and quality gates that make iteration safer." + ], + "avoid": [ + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim.", + "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here." + ], + "readme_excerpt": "# Superpowers\n\nSuperpowers is a complete software development methodology for your coding agents, built on top of a set of composable skills and some initial instructions that make sure your agent uses them.\n\n## Quickstart\n\nGive your agent Superpowers: [Claude Code](#claude-code), [Codex CLI](#codex-cli), [Codex App](#codex-app), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [OpenCode](#opencode), [Cursor](#cursor), [GitHub Copilot CLI](#github-copilot-cli).\n\n## How it works\n\nIt starts from the moment you fire up your coding agent. As soon as it sees that you're building something, it *doesn't* just jump into trying to write code. Instead, it steps back and asks you what you're really trying to do. \n\nOnce it's teased a spec out of the conversation, it shows it to you in chunks short enough to actually read and digest. \n\nAfter you've signed off on the design, your agent puts together an implementation plan that's clear enough for an enthusiastic junior engineer with poor taste, no judgement, no project context, and an aversion to testing to follow. It emphasizes true red/green TDD, YAGNI (You Aren't Gonna Need It), and DRY. \n\nNext up, once you say \"go\", it launches a *" + }, + { + "full_name": "affaan-m/ECC", + "html_url": "https://github.com/affaan-m/ECC", + "description": "The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.", + "stars": 214381, + "topics": [ + "ai-agents", + "anthropic", + "claude", + "claude-code", + "developer-tools", + "llm", + "mcp", + "productivity" + ], + "primary_language": "JavaScript", + "patterns": [ + { + "category": "method", + "matched": "workflow", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale." + }, + { + "category": "execution", + "matched": "command, shell, script", + "borrow": "Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not copy repo-specific commands or environment assumptions verbatim." + }, + { + "category": "structure", + "matched": "guide, docs", + "borrow": "Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not mirror documentation bulk that adds context cost without improving reliability." + }, + { + "category": "governance", + "matched": "security, review", + "borrow": "Borrow the explicit review, safety, or operational trust boundaries.", + "avoid": "Do not import compliance-heavy process where the new skill does not need it." + } + ], + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure.", + "Borrow the way it separates explanation, examples, and reusable structure." + ], + "avoid": [ + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim.", + "Do not mirror documentation bulk that adds context cost without improving reliability." + ], + "readme_excerpt": "**Language:** English | [Português (Brasil)](docs/pt-BR/README.md) | [简体中文](README.zh-CN.md) | [繁體中文](docs/zh-TW/README.md) | [日本語](docs/ja-JP/README.md) | [한국어](docs/ko-KR/README.md) | [Türkçe](docs/tr/README.md) | [Русский](docs/ru/README.md) | [Tiếng Việt](docs/vi-VN/README.md) | [ไทย](docs/th/README.md) | [Deutsch](docs/de-DE/README.md) | [Español](docs/es/README.md)\n\n![ECC — the agent harness operating system](assets/hero.png)\n\n[![Stars](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fstars&style=flat)](https://github.com/affaan-m/ECC/stargazers)\n[![Forks](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fforks&style=flat)](https://github.com/affaan-m/ECC/network/members)\n[![Contributors](https://img.shields.io/github/contributors/affaan-m/ECC?style=flat)](https://github.com/affaan-m/ECC/graphs/contributors)\n[![npm ecc-universal](https://img.shields.io/npm/dw/ecc-universal?label=ecc-universal%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-universal)\n[![npm ecc-agentshield](https://img.shields.io/npm/dw/ecc-agentshield?label=ecc-agentshield%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-agen" + }, + { + "full_name": "multica-ai/andrej-karpathy-skills", + "html_url": "https://github.com/multica-ai/andrej-karpathy-skills", + "description": "A single CLAUDE.md file to improve Claude Code behavior, derived from Andrej Karpathy's observations on LLM coding pitfalls.", + "stars": 174264, + "topics": [], + "primary_language": null, + "patterns": [ + { + "category": "evaluation", + "matched": "test, validation", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here." + }, + { + "category": "structure", + "matched": "guide", + "borrow": "Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not mirror documentation bulk that adds context cost without improving reliability." + }, + { + "category": "portability", + "matched": "plugin", + "borrow": "Borrow how it keeps core semantics stable across environments or integrations.", + "avoid": "Do not inherit platform lock-in or vendor-specific branching unless truly required." + } + ], + "borrow": [ + "Borrow explicit validation and quality gates that make iteration safer.", + "Borrow the way it separates explanation, examples, and reusable structure.", + "Borrow how it keeps core semantics stable across environments or integrations." + ], + "avoid": [ + "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "Do not mirror documentation bulk that adds context cost without improving reliability.", + "Do not inherit platform lock-in or vendor-specific branching unless truly required." + ], + "readme_excerpt": "# Karpathy-Inspired Claude Code Guidelines\n\n> Check out my new project [Multica](https://github.com/multica-ai/multica) — an open-source platform for running and managing coding agents with reusable skills.\n>\n> Follow me on X: [https://x.com/jiayuan_jy](https://x.com/jiayuan_jy)\n\nA single `CLAUDE.md` file to improve Claude Code behavior, derived from [Andrej Karpathy's observations](https://x.com/karpathy/status/2015883857489522876) on LLM coding pitfalls.\n\nEnglish | [简体中文](./README.zh.md)\n\n## The Problems\n\nFrom Andrej's post:\n\n> \"The models make wrong assumptions on your behalf and just run along with them without checking. They don't manage their confusion, don't seek clarifications, don't surface inconsistencies, don't present tradeoffs, don't push back when they should.\"\n\n> \"They really like to overcomplicate code and APIs, bloat abstractions, don't clean up dead code... implement a bloated construction over 1000 lines when 100 would do.\"\n\n> \"They still sometimes change/remove comments and code they don't sufficiently understand as side effects, even if orthogonal to the task.\"\n\n## The Solution\n\nFour principles in one file that directly address these issues:\n\n| Principle | Addr" + } + ], + "cross_repo": { + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure.", + "Borrow explicit validation and quality gates that make iteration safer.", + "Borrow the way it separates explanation, examples, and reusable structure.", + "Borrow how it keeps core semantics stable across environments or integrations.", + "Ask the user which of these patterns feels most worth borrowing before freezing the package shape." + ], + "avoid": [ + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim.", + "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "Do not mirror documentation bulk that adds context cost without improving reliability.", + "Do not inherit platform lock-in or vendor-specific branching unless truly required." + ] + }, + "external_references": [ + { + "name": "obra/superpowers", + "category": "method", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "source": "external" + }, + { + "name": "affaan-m/ECC", + "category": "method", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "source": "external" + }, + { + "name": "multica-ai/andrej-karpathy-skills", + "category": "evaluation", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "source": "external" + } + ], + "borrow_prompt": "I found 3 public GitHub projects worth studying. Do you want to borrow any of these patterns for method, structure, execution, or portability?", + "warnings": [] +} diff --git a/reports/github-benchmark-scan.md b/reports/github-benchmark-scan.md new file mode 100644 index 00000000..24abb4f7 --- /dev/null +++ b/reports/github-benchmark-scan.md @@ -0,0 +1,166 @@ +# GitHub Benchmark Scan + +- Query: `skills` +- Source: `github-api` +- Top repositories: `3` + +## Suggested Next Step + +Review the 3 benchmark objects below, then decide whether any of their patterns are worth borrowing into the new skill. + +## Top 3 Benchmark Repositories + +### obra/superpowers +- URL: https://github.com/obra/superpowers +- Stars: `226125` +- Description: An agentic skills framework & software development methodology that works. +- Topics: None + +#### Patterns worth studying + +- **method** (`process`): Borrow the way it turns a messy workflow into a repeatable operating path. +- **execution** (`cli`): Borrow the clear execution entrypoints and command structure. +- **evaluation** (`test`): Borrow explicit validation and quality gates that make iteration safer. +- **structure** (`docs`): Borrow the way it separates explanation, examples, and reusable structure. + +#### Borrow + +- Borrow the way it turns a messy workflow into a repeatable operating path. +- Borrow the clear execution entrypoints and command structure. +- Borrow explicit validation and quality gates that make iteration safer. + +#### Avoid + +- Do not import process overhead that only exists for that project's scale. +- Do not copy repo-specific commands or environment assumptions verbatim. +- Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here. + +#### README glimpse + +# Superpowers + +Superpowers is a complete software development methodology for your coding agents, built on top of a set of composable skills and some initial instructions that make sure your agent uses them. + +## Quickstart + +Give your agent Superpowers: [Claude Code](#claude-code), [Codex CLI](#codex-cli), [Codex App](#codex-app), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [OpenCode](#opencode), [Cursor](#cursor), [GitHub Copilot CLI](#github-copilot-cli). + +## How it works + +It starts from the moment you fire up your coding agent. As soon as it sees that you're building something, it *doesn't* just jump into trying to write code. Instead, it steps back and asks you what you're really trying to do. + +Once it's teased a spec out of the conversation, it shows it to you in chunks short enough to actually read and digest. + +After you've signed off on the design, your agent puts together an implementation plan that's clear enough for an enthusiastic junior engineer with poor taste, no judgement, no project context, and an aversion to testing to follow. It emphasizes true red/green TDD, YAGNI (You Aren't Gonna Need It), and DRY. + +Next up, once you say "go", it launches a * + +### affaan-m/ECC +- URL: https://github.com/affaan-m/ECC +- Stars: `214381` +- Description: The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond. +- Topics: ai-agents, anthropic, claude, claude-code, developer-tools, llm, mcp, productivity + +#### Patterns worth studying + +- **method** (`workflow`): Borrow the way it turns a messy workflow into a repeatable operating path. +- **execution** (`command, shell, script`): Borrow the clear execution entrypoints and command structure. +- **structure** (`guide, docs`): Borrow the way it separates explanation, examples, and reusable structure. +- **governance** (`security, review`): Borrow the explicit review, safety, or operational trust boundaries. + +#### Borrow + +- Borrow the way it turns a messy workflow into a repeatable operating path. +- Borrow the clear execution entrypoints and command structure. +- Borrow the way it separates explanation, examples, and reusable structure. + +#### Avoid + +- Do not import process overhead that only exists for that project's scale. +- Do not copy repo-specific commands or environment assumptions verbatim. +- Do not mirror documentation bulk that adds context cost without improving reliability. + +#### README glimpse + +**Language:** English | [Português (Brasil)](docs/pt-BR/README.md) | [简体中文](README.zh-CN.md) | [繁體中文](docs/zh-TW/README.md) | [日本語](docs/ja-JP/README.md) | [한국어](docs/ko-KR/README.md) | [Türkçe](docs/tr/README.md) | [Русский](docs/ru/README.md) | [Tiếng Việt](docs/vi-VN/README.md) | [ไทย](docs/th/README.md) | [Deutsch](docs/de-DE/README.md) | [Español](docs/es/README.md) + +![ECC — the agent harness operating system](assets/hero.png) + +[![Stars](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fstars&style=flat)](https://github.com/affaan-m/ECC/stargazers) +[![Forks](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fforks&style=flat)](https://github.com/affaan-m/ECC/network/members) +[![Contributors](https://img.shields.io/github/contributors/affaan-m/ECC?style=flat)](https://github.com/affaan-m/ECC/graphs/contributors) +[![npm ecc-universal](https://img.shields.io/npm/dw/ecc-universal?label=ecc-universal%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-universal) +[![npm ecc-agentshield](https://img.shields.io/npm/dw/ecc-agentshield?label=ecc-agentshield%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-agen + +### multica-ai/andrej-karpathy-skills +- URL: https://github.com/multica-ai/andrej-karpathy-skills +- Stars: `174264` +- Description: A single CLAUDE.md file to improve Claude Code behavior, derived from Andrej Karpathy's observations on LLM coding pitfalls. +- Topics: None + +#### Patterns worth studying + +- **evaluation** (`test, validation`): Borrow explicit validation and quality gates that make iteration safer. +- **structure** (`guide`): Borrow the way it separates explanation, examples, and reusable structure. +- **portability** (`plugin`): Borrow how it keeps core semantics stable across environments or integrations. + +#### Borrow + +- Borrow explicit validation and quality gates that make iteration safer. +- Borrow the way it separates explanation, examples, and reusable structure. +- Borrow how it keeps core semantics stable across environments or integrations. + +#### Avoid + +- Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here. +- Do not mirror documentation bulk that adds context cost without improving reliability. +- Do not inherit platform lock-in or vendor-specific branching unless truly required. + +#### README glimpse + +# Karpathy-Inspired Claude Code Guidelines + +> Check out my new project [Multica](https://github.com/multica-ai/multica) — an open-source platform for running and managing coding agents with reusable skills. +> +> Follow me on X: [https://x.com/jiayuan_jy](https://x.com/jiayuan_jy) + +A single `CLAUDE.md` file to improve Claude Code behavior, derived from [Andrej Karpathy's observations](https://x.com/karpathy/status/2015883857489522876) on LLM coding pitfalls. + +English | [简体中文](./README.zh.md) + +## The Problems + +From Andrej's post: + +> "The models make wrong assumptions on your behalf and just run along with them without checking. They don't manage their confusion, don't seek clarifications, don't surface inconsistencies, don't present tradeoffs, don't push back when they should." + +> "They really like to overcomplicate code and APIs, bloat abstractions, don't clean up dead code... implement a bloated construction over 1000 lines when 100 would do." + +> "They still sometimes change/remove comments and code they don't sufficiently understand as side effects, even if orthogonal to the task." + +## The Solution + +Four principles in one file that directly address these issues: + +| Principle | Addr + +## Cross-Repo Borrow Recommendations + +- Borrow the way it turns a messy workflow into a repeatable operating path. +- Borrow the clear execution entrypoints and command structure. +- Borrow explicit validation and quality gates that make iteration safer. +- Borrow the way it separates explanation, examples, and reusable structure. +- Borrow how it keeps core semantics stable across environments or integrations. +- Ask the user which of these patterns feels most worth borrowing before freezing the package shape. + +## Cross-Repo Avoid Recommendations + +- Do not import process overhead that only exists for that project's scale. +- Do not copy repo-specific commands or environment assumptions verbatim. +- Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here. +- Do not mirror documentation bulk that adds context cost without improving reliability. +- Do not inherit platform lock-in or vendor-specific branching unless truly required. + +## Borrow Prompt + +I found 3 public GitHub projects worth studying. Do you want to borrow any of these patterns for method, structure, execution, or portability? diff --git a/reports/intent-dialogue.json b/reports/intent-dialogue.json new file mode 100644 index 00000000..eb483484 --- /dev/null +++ b/reports/intent-dialogue.json @@ -0,0 +1,79 @@ +{ + "skill_name": "yao-meta-skill", + "title": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "focus": "portability-and-contract", + "opening_frame": "Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that.", + "reference_note": "If you already have examples you admire, bring them in. We will learn the pattern, not copy the source.", + "conversation_path": [ + "Start with the user's own words, not package vocabulary.", + "Reflect the job, output, and non-goals back in one clean sentence.", + "Only then offer a tiny scaffold if it would help the user move faster." + ], + "opening_styles": [ + { + "label": "温柔陪伴型", + "best_when": "用户想法还散、还在试探，或者需要先被接住。", + "message": "我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。" + }, + { + "label": "专业教练型", + "best_when": "用户目标比较明确，希望被高效带着走。", + "message": "我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。" + }, + { + "label": "共创伙伴型", + "best_when": "用户已经有一些想法，希望一起打磨，不想被填表。", + "message": "我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。" + } + ], + "optional_scaffold": [ + "The repeated job it should reliably handle", + "The real inputs people will hand to it", + "The useful output it should hand back", + "What it should clearly refuse" + ], + "questions": [ + { + "question": "If this skill worked beautifully, what recurring job would it reliably handle for the user every time?", + "why": "This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape." + }, + { + "question": "When someone reaches for this skill in the real world, what materials will they actually hand to it?", + "why": "Input shape decides whether references, scripts, or templates are needed." + }, + { + "question": "What finished output should it hand back so the user can immediately keep moving?", + "why": "Outputs should drive the package structure before extra guidance is added." + }, + { + "question": "Which nearby requests should this skill politely refuse so the boundary stays clean?", + "why": "The exclusion list is the fastest route to better trigger quality." + }, + { + "question": "What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?", + "why": "Constraints decide how much structure, packaging, and review the skill actually needs." + }, + { + "question": "Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example?", + "why": "A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material." + }, + { + "question": "Which environments or clients must be able to consume this skill?", + "why": "This sets the minimum metadata and degradation strategy." + } + ], + "output": { + "capability_sentence": "Yao Meta Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily.", + "required_capture": [ + "recurring job", + "real inputs", + "required outputs", + "exclusions", + "constraints", + "reference preferences", + "first evaluation target" + ], + "recommended_first_gate": "portability and contract" + } +} \ No newline at end of file diff --git a/reports/intent-dialogue.md b/reports/intent-dialogue.md new file mode 100644 index 00000000..a42b7762 --- /dev/null +++ b/reports/intent-dialogue.md @@ -0,0 +1,75 @@ +# Intent Dialogue + +Skill: `yao-meta-skill` + +## Opening Frame + +Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that. + +## Opening Tone Options + +### 温柔陪伴型 + +- Best when: 用户想法还散、还在试探，或者需要先被接住。 +- Example: 我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。 + +### 专业教练型 + +- Best when: 用户目标比较明确，希望被高效带着走。 +- Example: 我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。 + +### 共创伙伴型 + +- Best when: 用户已经有一些想法，希望一起打磨，不想被填表。 +- Example: 我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。 + +## Conversation Path + +1. Start with the user's own words, not package vocabulary. +2. Reflect the job, output, and non-goals back in one clean sentence. +3. Only then offer a tiny scaffold if it would help the user move faster. + +## Why Start Here + +Use this short dialogue before deep authoring. The goal is to learn the real job, output, exclusions, and constraints so the first package is small but accurate. + +## Current Anchor + +- Title: `Yao Meta Skill` +- Description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. +- Focus: `portability-and-contract` +- Reference note: If you already have examples you admire, bring them in. We will learn the pattern, not copy the source. + +## Questions To Ask + +1. If this skill worked beautifully, what recurring job would it reliably handle for the user every time? + Why: This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape. +2. When someone reaches for this skill in the real world, what materials will they actually hand to it? + Why: Input shape decides whether references, scripts, or templates are needed. +3. What finished output should it hand back so the user can immediately keep moving? + Why: Outputs should drive the package structure before extra guidance is added. +4. Which nearby requests should this skill politely refuse so the boundary stays clean? + Why: The exclusion list is the fastest route to better trigger quality. +5. What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit? + Why: Constraints decide how much structure, packaging, and review the skill actually needs. +6. Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example? + Why: A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material. +7. Which environments or clients must be able to consume this skill? + Why: This sets the minimum metadata and degradation strategy. + +## Capture Before Drafting + +- Capability sentence: Yao Meta Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily. +- Recommended first gate: `portability and contract` +- Tiny optional scaffold: + - The repeated job it should reliably handle + - The real inputs people will hand to it + - The useful output it should hand back + - What it should clearly refuse +- Capture: `recurring job` +- Capture: `real inputs` +- Capture: `required outputs` +- Capture: `exclusions` +- Capture: `constraints` +- Capture: `reference preferences` +- Capture: `first evaluation target` diff --git a/reports/iteration-directions.json b/reports/iteration-directions.json index 81c26411..e376d9ed 100644 --- a/reports/iteration-directions.json +++ b/reports/iteration-directions.json @@ -4,11 +4,24 @@ "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "maturity_tier": "governed", "selection_rule": "Pick the three smallest next steps that increase reliability more than they increase context cost.", - "recommended_now": "Harden portability semantics", - "recommended_now_why": "The skill already signals reuse across environments, so contract clarity matters early.", + "recommended_now": "Borrow one proven pattern on purpose", + "recommended_now_why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", "defer_for_now": "Create an iteration evidence loop" }, "directions": [ + { + "priority": 2, + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design.", + "do_now": "Do this first.", + "wait_on": "Wait to add broader structure until this move clearly improves reliability." + }, { "priority": 5, "title": "Harden portability semantics", @@ -19,11 +32,11 @@ "Package the skill once to verify adapter expectations." ], "unlocks": "Safer cross-environment reuse with less target drift.", - "do_now": "Do this first.", - "wait_on": "Wait to add broader structure until this move clearly improves reliability." + "do_now": "Do this after the first move lands cleanly.", + "wait_on": "Wait until the package has evidence that this extra structure is justified." }, { - "priority": 11, + "priority": 12, "title": "Create an iteration evidence loop", "why": "The package should show what changed and why after the first draft.", "actions": [ diff --git a/reports/iteration-directions.md b/reports/iteration-directions.md index c8b48f9c..258b6200 100644 --- a/reports/iteration-directions.md +++ b/reports/iteration-directions.md @@ -4,30 +4,47 @@ Skill: `yao-meta-skill` - Maturity tier: `governed` - Selection rule: Pick the three smallest next steps that increase reliability more than they increase context cost. -- Start here: `Harden portability semantics` -- Why first: The skill already signals reuse across environments, so contract clarity matters early. +- Start here: `Borrow one proven pattern on purpose` +- Why first: You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely. - Defer for now: `Create an iteration evidence loop` ## Top 3 Next Moves +### 2. Borrow one proven pattern on purpose + +- Why now: You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely. +- Timing: Do this first. +- Recommended actions: + - Read the strongest pattern from obra/superpowers. +- Unlocks: A cleaner package shape with less accidental over-design. +- Wait on: Wait to add broader structure until this move clearly improves reliability. + + - Decide whether to borrow method, structure, execution, or portability, but only one of them first. +- Unlocks: A cleaner package shape with less accidental over-design. +- Wait on: Wait to add broader structure until this move clearly improves reliability. + + - Record what you will not borrow so the package stays light. +- Unlocks: A cleaner package shape with less accidental over-design. +- Wait on: Wait to add broader structure until this move clearly improves reliability. + ### 5. Harden portability semantics - Why now: The skill already signals reuse across environments, so contract clarity matters early. -- Timing: Do this first. +- Timing: Do this after the first move lands cleanly. - Recommended actions: - Confirm activation mode, execution context, and trust assumptions. - Unlocks: Safer cross-environment reuse with less target drift. -- Wait on: Wait to add broader structure until this move clearly improves reliability. +- Wait on: Wait until the package has evidence that this extra structure is justified. - Add or review degradation strategy for non-native targets. - Unlocks: Safer cross-environment reuse with less target drift. -- Wait on: Wait to add broader structure until this move clearly improves reliability. +- Wait on: Wait until the package has evidence that this extra structure is justified. - Package the skill once to verify adapter expectations. - Unlocks: Safer cross-environment reuse with less target drift. -- Wait on: Wait to add broader structure until this move clearly improves reliability. +- Wait on: Wait until the package has evidence that this extra structure is justified. -### 11. Create an iteration evidence loop +### 12. Create an iteration evidence loop - Why now: The package should show what changed and why after the first draft. - Timing: Do this after the first move lands cleanly. diff --git a/reports/output-risk-profile.json b/reports/output-risk-profile.json new file mode 100644 index 00000000..5f5a1269 --- /dev/null +++ b/reports/output-risk-profile.json @@ -0,0 +1,143 @@ +{ + "skill_name": "yao-meta-skill", + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "top_risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap." + ], + "output_constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output." + ], + "self_repair_checks": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards.", + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note.", + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ], + "reviewer_note": "Use this report before deepening the package and again before approving example outputs." +} diff --git a/reports/output-risk-profile.md b/reports/output-risk-profile.md new file mode 100644 index 00000000..88f3bdd1 --- /dev/null +++ b/reports/output-risk-profile.md @@ -0,0 +1,60 @@ +# Output Risk Profile + +Skill: `yao-meta-skill` + +## Why This Exists + +Generated skills often fail in small output details: generic headings, cluttered citations, fragile screenshots, weak Markdown rendering, or missing execution assumptions. This profile predicts the most likely output mistakes before the skill is used heavily. + +## Matched Risk Families + +### Markdown readability +- Matched keywords: md, report, doc +- Score: `3` + +### Citation and footnote clutter +- Matched keywords: source, reference +- Score: `2` + +### Screenshot and visual capture +- Matched keywords: capture +- Score: `1` + +### Code and command safety +- Matched keywords: script +- Score: `1` + +### Tone and specificity +- Matched keywords: summary +- Score: `1` + +## Likely Output Mistakes + +- Tables can render as dense grids with weak hierarchy or poor mobile readability. +- Long bullets can make the output look complete while hiding the actual decision logic. +- Footnote markers or dense citation notes can interrupt the reading flow. +- Evidence can be over-attached to obvious statements and under-attached to risky claims. +- Screenshots can be captured from the wrong state, wrong viewport, or wrong crop. +- Missing screenshots can cause the skill to invent visual references instead of declaring the gap. + +## Output Constraints To Apply + +- Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets. +- Keep table cells short and move explanations below the table. +- Attach citations only to claims that need evidence, not to every sentence. +- Group source notes at the end of a section when inline markers would hurt readability. +- Never invent a screenshot; state when visual evidence is missing. +- Record the source, viewport, and crop intent for any screenshot-dependent output. + +## Self-Repair Checks + +- Preview whether each table still reads well when columns are narrow. +- Convert any table with paragraph-length cells into bullets or cards. +- Remove decorative citations that do not support a material claim. +- Move repeated source explanations into one compact source note. +- Check that every screenshot reference points to a real provided or generated asset. +- Reword any visual instruction that depends on an unseen screen state. + +## Reviewer Note + +Use this report before deepening the package and again before approving example outputs. diff --git a/reports/output_quality_scorecard.json b/reports/output_quality_scorecard.json new file mode 100644 index 00000000..94191d2d --- /dev/null +++ b/reports/output_quality_scorecard.json @@ -0,0 +1,482 @@ +{ + "ok": true, + "cases": "evals/output/cases.jsonl", + "summary": { + "case_count": 3, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "gate_pass": true, + "failure_taxonomy": [] + }, + "results": [ + { + "id": "skill-package-contract", + "prompt": "Turn this repeated workflow into a reusable team skill package.", + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "checks": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0, + "failure_type": "missing_entrypoint", + "passed": false, + "missing": [ + "SKILL.md" + ], + "present_forbidden": [] + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0, + "failure_type": "missing_interface", + "passed": false, + "missing": [ + "agents/interface.yaml" + ], + "present_forbidden": [] + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0, + "failure_type": "missing_review_artifact", + "passed": false, + "missing": [ + "reports/skill-overview.html" + ], + "present_forbidden": [] + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0, + "failure_type": "resource_boundary", + "passed": false, + "missing": [ + "references/", + "scripts/", + "reports/" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0, + "failure_type": "missing_entrypoint", + "passed": false, + "missing": [ + "SKILL.md" + ], + "present_forbidden": [] + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0, + "failure_type": "missing_interface", + "passed": false, + "missing": [ + "agents/interface.yaml" + ], + "present_forbidden": [] + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0, + "failure_type": "missing_review_artifact", + "passed": false, + "missing": [ + "reports/skill-overview.html" + ], + "present_forbidden": [] + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0, + "failure_type": "resource_boundary", + "passed": false, + "missing": [ + "references/", + "scripts/", + "reports/" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "checks": [ + { + "id": "has-entrypoint", + "description": "Output names the routeable Skill entrypoint.", + "weight": 1.0, + "failure_type": "missing_entrypoint", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-interface", + "description": "Output includes neutral interface metadata.", + "weight": 1.0, + "failure_type": "missing_interface", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-report", + "description": "Output points reviewer to generated overview report.", + "weight": 1.0, + "failure_type": "missing_review_artifact", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-resource-boundary", + "description": "Output preserves entrypoint/resource separation.", + "weight": 1.0, + "failure_type": "resource_boundary", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + }, + { + "id": "output-eval-expectation", + "prompt": "Upgrade this production skill so we know whether the generated output is better than baseline.", + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "checks": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0, + "failure_type": "missing_baseline_delta", + "passed": false, + "missing": [ + "with-skill", + "baseline" + ], + "present_forbidden": [] + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0, + "failure_type": "missing_assertions", + "passed": false, + "missing": [ + "assertion grading" + ], + "present_forbidden": [] + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0, + "failure_type": "missing_scorecard", + "passed": false, + "missing": [ + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0, + "failure_type": "missing_failure_taxonomy", + "passed": false, + "missing": [ + "failure taxonomy" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0, + "failure_type": "missing_baseline_delta", + "passed": false, + "missing": [ + "with-skill", + "baseline" + ], + "present_forbidden": [] + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0, + "failure_type": "missing_assertions", + "passed": false, + "missing": [ + "assertion grading" + ], + "present_forbidden": [] + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0, + "failure_type": "missing_scorecard", + "passed": false, + "missing": [ + "reports/output_quality_scorecard.md" + ], + "present_forbidden": [] + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0, + "failure_type": "missing_failure_taxonomy", + "passed": false, + "missing": [ + "failure taxonomy" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "checks": [ + { + "id": "has-baseline-comparison", + "description": "Output explicitly compares with-skill and baseline outputs.", + "weight": 1.0, + "failure_type": "missing_baseline_delta", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-assertion-grading", + "description": "Output includes objective assertion grading.", + "weight": 1.0, + "failure_type": "missing_assertions", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-scorecard", + "description": "Output produces a scorecard report path.", + "weight": 1.0, + "failure_type": "missing_scorecard", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-failure-taxonomy", + "description": "Output records failure taxonomy.", + "weight": 1.0, + "failure_type": "missing_failure_taxonomy", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + }, + { + "id": "ir-before-packaging", + "prompt": "Package this skill for OpenAI, Claude, Agent Skills, and generic targets.", + "baseline": { + "score": 0.0, + "passed_count": 0, + "failed_count": 4, + "checks": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0, + "failure_type": "missing_ir", + "passed": false, + "missing": [ + "Skill IR first" + ], + "present_forbidden": [] + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0, + "failure_type": "incomplete_ir_contract", + "passed": false, + "missing": [ + "job_to_be_done", + "trigger_surface", + "eval_plan", + "governance" + ], + "present_forbidden": [] + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0, + "failure_type": "missing_targets", + "passed": false, + "missing": [ + "OpenAI", + "Claude", + "Agent Skills", + "generic" + ], + "present_forbidden": [] + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0, + "failure_type": "semantic_drift", + "passed": false, + "missing": [ + "preserve the same capability contract" + ], + "present_forbidden": [] + } + ], + "failed": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0, + "failure_type": "missing_ir", + "passed": false, + "missing": [ + "Skill IR first" + ], + "present_forbidden": [] + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0, + "failure_type": "incomplete_ir_contract", + "passed": false, + "missing": [ + "job_to_be_done", + "trigger_surface", + "eval_plan", + "governance" + ], + "present_forbidden": [] + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0, + "failure_type": "missing_targets", + "passed": false, + "missing": [ + "OpenAI", + "Claude", + "Agent Skills", + "generic" + ], + "present_forbidden": [] + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0, + "failure_type": "semantic_drift", + "passed": false, + "missing": [ + "preserve the same capability contract" + ], + "present_forbidden": [] + } + ] + }, + "with_skill": { + "score": 100.0, + "passed_count": 4, + "failed_count": 0, + "checks": [ + { + "id": "has-ir-first", + "description": "Output requires Skill IR before packaging.", + "weight": 1.0, + "failure_type": "missing_ir", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-core-ir-fields", + "description": "Output lists core IR contract fields.", + "weight": 1.0, + "failure_type": "incomplete_ir_contract", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-targets", + "description": "Output names requested runtime targets.", + "weight": 1.0, + "failure_type": "missing_targets", + "passed": true, + "missing": [], + "present_forbidden": [] + }, + { + "id": "has-semantic-preservation", + "description": "Output says targets must preserve the capability contract.", + "weight": 1.0, + "failure_type": "semantic_drift", + "passed": true, + "missing": [], + "present_forbidden": [] + } + ], + "failed": [] + }, + "delta": 100.0, + "winner": "with_skill", + "failure_taxonomy": [] + } + ], + "failures": [], + "artifacts": { + "json": "reports/output_quality_scorecard.json", + "markdown": "reports/output_quality_scorecard.md" + } +} diff --git a/reports/output_quality_scorecard.md b/reports/output_quality_scorecard.md new file mode 100644 index 00000000..31ed67f2 --- /dev/null +++ b/reports/output_quality_scorecard.md @@ -0,0 +1,28 @@ +# Output Quality Scorecard + +This v0 scorecard compares static without-skill and with-skill outputs using assertion grading. + +- Cases: `3` +- Baseline pass rate: `0.0` +- With-skill pass rate: `100.0` +- Delta: `100.0` +- Regressions: `0` +- Gate pass: `True` + +## Case Results + +| Case | Baseline | With Skill | Delta | Winner | Failed With-Skill Assertions | +| --- | ---: | ---: | ---: | --- | --- | +| skill-package-contract | 0.0 | 100.0 | 100.0 | with_skill | None | +| output-eval-expectation | 0.0 | 100.0 | 100.0 | with_skill | None | +| ir-before-packaging | 0.0 | 100.0 | 100.0 | with_skill | None | + +## Failure Taxonomy + +- No with-skill assertion failures. + +## Next Fixes + +- Add holdout cases before using this as a release gate. +- Promote repeated failed assertions into the output-risk profile. +- Keep assertions tied to material deliverables, not phrasing trivia. diff --git a/reports/reference-scan.json b/reports/reference-scan.json new file mode 100644 index 00000000..93bdcd73 --- /dev/null +++ b/reports/reference-scan.json @@ -0,0 +1,40 @@ +{ + "skill_name": "yao-meta-skill", + "title": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "scan_focus": [ + { + "label": "Evaluation pattern", + "reason": "This skill already carries eval assets, so benchmark how top examples define trigger boundaries and quality gates." + }, + { + "label": "Execution pattern", + "reason": "There is deterministic logic in scripts, so compare how strong references separate prose from executable steps." + }, + { + "label": "Portability pattern", + "reason": "The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source." + }, + { + "label": "Method pattern", + "reason": "Use the core job description as the anchor for comparison: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + } + ], + "external_references": [], + "user_references": [], + "local_constraints": [], + "borrow_plan": [ + "External benchmark first: let high-quality public references define the upper bound for method, structure, execution, or portability.", + "User references second: use them to understand taste, standards, and directional preferences without copying source phrasing.", + "Local fit third: use local assets only to detect naming conflicts, private dependencies, or compatibility constraints.", + "Borrow patterns, not prose: extract loops, boundaries, metadata, and operator flow without copying source-specific language.", + "Keep the package light: reject any borrowed pattern that increases context cost faster than it increases reliability." + ], + "priority_rule": "External benchmark objects set the pattern ceiling. User references refine taste and standards. Local files only calibrate fit, risk, and compatibility.", + "non_goals": [ + "Do not copy source prose or branding into the new skill.", + "Do not import gates that cost more context than they save.", + "Do not use benchmark scanning to justify scope creep.", + "Do not let local historical habits outrank stronger public benchmarks." + ] +} \ No newline at end of file diff --git a/reports/reference-scan.md b/reports/reference-scan.md new file mode 100644 index 00000000..f5148a3b --- /dev/null +++ b/reports/reference-scan.md @@ -0,0 +1,55 @@ +# Reference Scan + +Skill: `yao-meta-skill` + +## Why This Step Exists + +Use a short benchmark pass before authoring the package in depth. External benchmark objects should define the pattern ceiling. Local files are used afterward only to calibrate fit, privacy, naming, and compatibility. + +## Current Skill Anchor + +- Title: `Yao Meta Skill` +- Description: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. + +## Scan Focus + +- **Evaluation pattern**: This skill already carries eval assets, so benchmark how top examples define trigger boundaries and quality gates. +- **Execution pattern**: There is deterministic logic in scripts, so compare how strong references separate prose from executable steps. +- **Portability pattern**: The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source. +- **Method pattern**: Use the core job description as the anchor for comparison: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse. + +## Priority Rule + +- External benchmark objects set the pattern ceiling. User references refine taste and standards. Local files only calibrate fit, risk, and compatibility. + +## External Benchmark Objects + +- No explicit external benchmark objects recorded yet. +- Recommended: capture 2 to 5 external references at most. +- Suggested mix: one method reference, one structure reference, one execution or portability reference. + +## User-Supplied References + +- No user-supplied references recorded yet. +- Ask whether the user has a repo, product, page, workflow, or prompt example worth learning from. +- Treat these as pattern references only, not as text to be copied. + +## Local Fit Check + +- No local constraints recorded yet. +- Use this section for naming collisions, private dependencies, compatibility limits, or existing library conventions. + +## Borrow Plan + +- External benchmark first: let high-quality public references define the upper bound for method, structure, execution, or portability. +- User references second: use them to understand taste, standards, and directional preferences without copying source phrasing. +- Local fit third: use local assets only to detect naming conflicts, private dependencies, or compatibility constraints. +- Borrow patterns, not prose: extract loops, boundaries, metadata, and operator flow without copying source-specific language. +- Keep the package light: reject any borrowed pattern that increases context cost faster than it increases reliability. + +## Non-Goals + +- Do not copy source prose or branding into the new skill. +- Do not import gates that cost more context than they save. +- Do not use benchmark scanning to justify scope creep. +- Do not let local historical habits outrank stronger public benchmarks. diff --git a/reports/reference-synthesis.json b/reports/reference-synthesis.json index 551ce645..03d6dada 100644 --- a/reports/reference-synthesis.json +++ b/reports/reference-synthesis.json @@ -6,7 +6,35 @@ "band": "low", "gate_passed": false }, - "github_benchmarks": [], + "github_benchmarks": [ + { + "name": "obra/superpowers", + "url": "https://github.com/obra/superpowers", + "stars": 226125, + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure." + ] + }, + { + "name": "affaan-m/ECC", + "url": "https://github.com/affaan-m/ECC", + "stars": 214381, + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure." + ] + }, + { + "name": "multica-ai/andrej-karpathy-skills", + "url": "https://github.com/multica-ai/andrej-karpathy-skills", + "stars": 174264, + "borrow": [ + "Borrow explicit validation and quality gates that make iteration safer.", + "Borrow the way it separates explanation, examples, and reusable structure." + ] + } + ], "source_tracks": [ { "source_type": "official", @@ -46,12 +74,16 @@ "borrow_now": [ "Borrow a first-time operator flow that explains itself before it asks for more structure.", "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", - "Borrow the discipline of defining what the skill should not own before growing the package." + "Borrow the discipline of defining what the skill should not own before growing the package.", + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure." ], "avoid_now": [ "Do not mimic product polish that adds UI bulk without improving clarity.", "Do not create experimental overhead that exceeds the skill's real risk tier.", - "Do not expand execution assets until route boundaries stay clean." + "Do not expand execution assets until route boundaries stay clean.", + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim." ], "quality_risers": [ "Use GitHub repositories for concrete package and workflow patterns.", @@ -61,7 +93,7 @@ ], "pattern_gate": { "threshold": 4, - "source_count": 3, + "source_count": 6, "accepted": [ { "name": "Official workflow product ergonomics", @@ -82,6 +114,46 @@ ], "missing": [], "score": 4 + }, + { + "name": "obra/superpowers", + "source_type": "github", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.; Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + }, + { + "name": "affaan-m/ECC", + "source_type": "github", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.; Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 } ], "deferred": [ @@ -126,9 +198,30 @@ "distinctiveness" ], "score": 3 + }, + { + "name": "multica-ai/andrej-karpathy-skills", + "source_type": "github", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.; Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "gates": { + "recurrence": true, + "generativity": false, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "distinctiveness", + "boundary" + ], + "missing": [ + "generativity" + ], + "score": 3 } ], - "summary": "1 accepted, 2 deferred using threshold 4/4." + "summary": "3 accepted, 3 deferred using threshold 4/4." }, "conflicts": [], "recommendation": { @@ -155,7 +248,7 @@ }, "decision_prompt": "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.", "source_mix": { - "github_benchmarks": 0, + "github_benchmarks": 3, "curated_tracks": 3, "user_references": 0 } diff --git a/reports/reference-synthesis.md b/reports/reference-synthesis.md index 7f05f399..8ade5ffa 100644 --- a/reports/reference-synthesis.md +++ b/reports/reference-synthesis.md @@ -6,7 +6,23 @@ Skill: `yao-meta-skill` ## Live GitHub Benchmarks -- No live GitHub benchmarks are attached yet. +### obra/superpowers +- URL: https://github.com/obra/superpowers +- Stars: `226125` +- Borrow: Borrow the way it turns a messy workflow into a repeatable operating path. +- Borrow: Borrow the clear execution entrypoints and command structure. + +### affaan-m/ECC +- URL: https://github.com/affaan-m/ECC +- Stars: `214381` +- Borrow: Borrow the way it turns a messy workflow into a repeatable operating path. +- Borrow: Borrow the clear execution entrypoints and command structure. + +### multica-ai/andrej-karpathy-skills +- URL: https://github.com/multica-ai/andrej-karpathy-skills +- Stars: `174264` +- Borrow: Borrow explicit validation and quality gates that make iteration safer. +- Borrow: Borrow the way it separates explanation, examples, and reusable structure. ## Curated World-Class Pattern Tracks @@ -36,22 +52,29 @@ Skill: `yao-meta-skill` - Borrow a first-time operator flow that explains itself before it asks for more structure. - Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. - Borrow the discipline of defining what the skill should not own before growing the package. +- Borrow the way it turns a messy workflow into a repeatable operating path. +- Borrow the clear execution entrypoints and command structure. ## Avoid Now - Do not mimic product polish that adds UI bulk without improving clarity. - Do not create experimental overhead that exceeds the skill's real risk tier. - Do not expand execution assets until route boundaries stay clean. +- Do not import process overhead that only exists for that project's scale. +- Do not copy repo-specific commands or environment assumptions verbatim. ## Pattern Gate -- Summary: 1 accepted, 2 deferred using threshold 4/4. +- Summary: 3 accepted, 3 deferred using threshold 4/4. - Acceptance threshold: `4/4` - Accepted patterns: - **Official workflow product ergonomics**: 4/4 (recurrence, generativity, distinctiveness, boundary) + - **obra/superpowers**: 4/4 (recurrence, generativity, distinctiveness, boundary) + - **affaan-m/ECC**: 4/4 (recurrence, generativity, distinctiveness, boundary) - Deferred patterns: - **Hypothesis-test-learn loop**: missing distinctiveness - **Boundary-first design**: missing distinctiveness + - **multica-ai/andrej-karpathy-skills**: missing generativity ## Default Recommendation diff --git a/reports/review-viewer.html b/reports/review-viewer.html new file mode 100644 index 00000000..2094b053 --- /dev/null +++ b/reports/review-viewer.html @@ -0,0 +1,376 @@ + + + + + + Yao Meta Skill Review Viewer + + + +
+
+

Review Viewer

+

Yao Meta Skill

+

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

+
+ maturity: governed + archetype: governed + format: agent-skills + updated: 2026-03-31 + intent confidence: 30 / 100 +
+
+ +
+

Architecture at a glance

+
Inputs
workflow, prompt, transcript, docs, or notes
Boundary
Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
Logic
Decide whether the request should become a skill and choose the lightest fit.; Capture job, output, exclusions, constraints, and standards.; Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
Usage
Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.; Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
Next
Review the top iteration directions before growing the package.
+
+ +
+
+

Core logic

+
  • Decide whether the request should become a skill and choose the lightest fit.
  • Capture job, output, exclusions, constraints, and standards.
  • Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.
  • Write the `description` early and test route quality before expanding the package.
  • Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.
+
+
+

How to use it

+
  • Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.
  • Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
+
+
+ +
+
+

Intent questions

+
  • If this skill worked beautifully, what recurring job would it reliably handle for the user every time?
    This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape.
  • When someone reaches for this skill in the real world, what materials will they actually hand to it?
    Input shape decides whether references, scripts, or templates are needed.
  • What finished output should it hand back so the user can immediately keep moving?
    Outputs should drive the package structure before extra guidance is added.
  • Which nearby requests should this skill politely refuse so the boundary stays clean?
    The exclusion list is the fastest route to better trigger quality.
  • What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?
    Constraints decide how much structure, packaging, and review the skill actually needs.
+
+
+

Why this package is strong

+
  • 触发面保持精简，并锚定在 frontmatter description。
  • 已打包 agents/interface.yaml，便于后续做跨平台适配。
  • 长指导被拆到 references 中，入口文件可以保持轻量。
  • 确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。
  • 包内包含可随 Skill 迁移的质量门禁或触发检查。
+
+
+ +
+
+

Borrow plan

+
  • No external benchmark objects recorded yet. Add 2 to 5 references before deepening the package.
+
+
+

Compare view

+

Winner: Current

{'priority': ['fewest false positives', 'fewest false negatives', 'highest near-neighbor pass rate', 'highest negative pass rate', 'highest precision', 'highest recall', 'shortest description']}

VariantTokensDev ErrorsHoldout ErrorsStrategy
Baseline800existing
Current6500current
Current6500current
+
+
+ +
+

Variant diff studio

+

Baseline

existing

Create and improve agent skills.

tokens 8 (0)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create and improve agent skills.

Drops from baseline

No dropped cues.

Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

tokens 65 (+57)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

Drops from baseline

  • Create and improve agent skills.

Winner — Current

current

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

tokens 65 (+57)dev 0 (0)holdout 0 (0)

Adds relative to baseline

  • Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.
  • Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

Drops from baseline

  • Create and improve agent skills.
+
+ +
+
+

Evidence readiness

+

Readiness score: 85/100

+
  • Intent clarity · needs review
    30/100 intent confidence.
  • Benchmark coverage · ready
    3 GitHub benchmark repositories attached.
  • Pattern gate · ready
    3 accepted, 3 deferred using threshold 4/4.
  • Conflict handling · ready
    No material conflicts detected.
  • Output risk profile · ready
    5 output risk families attached.
  • Artifact design profile · ready
    High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
  • Prompt quality profile · ready
    82.0/100 prompt-facing quality score.
+
+
+

Honest boundary check

+
    +
  • Are the known limits visible before the package deepens?
    • +
  • Does the evidence support the borrowed patterns?
    • +
  • Should uncertainty become a clarification question instead of more structure?
    • +
+
+
+ +
+
+

Output risk profile

+
  • Markdown readability
    Tables can render as dense grids with weak hierarchy or poor mobile readability.; Long bullets can make the output look complete while hiding the actual decision logic.
  • Citation and footnote clutter
    Footnote markers or dense citation notes can interrupt the reading flow.; Evidence can be over-attached to obvious statements and under-attached to risky claims.
  • Screenshot and visual capture
    Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.; Missing screenshots can cause the skill to invent visual references instead of declaring the gap.
+
+
+

Self-repair checks

+
  • Preview whether each table still reads well when columns are narrow.
  • Convert any table with paragraph-length cells into bullets or cards.
  • Remove decorative citations that do not support a material claim.
  • Move repeated source explanations into one compact source note.
  • Check that every screenshot reference points to a real provided or generated asset.
+
+
+ +
+
+

Artifact design profile

+

Design system: content-led editorial

+
  • Report or brief
    High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.
  • Code, CLI, or implementation guide
    Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.
+
+
+

Visual quality gates

+
  • Keep the first screen useful without requiring the reader to parse every detail.
  • Use tables only for comparisons; move explanations below the table.
  • Keep source notes readable without flooding the body with markers.
  • Name the working directory and required inputs for commands.
  • Mark destructive, networked, or external side-effect operations.
+
+
+ +
+
+

Prompt quality profile

+

Relevance: prompt-heavy · score 82.0 / 100 · complexity complex

+
  • Completeness · 80 / 100
    Name missing inputs, outputs, constraints, or success standards before deepening the package.
  • Clarity · 80 / 100
    Replace broad verbs with observable actions and define what done means.
  • Consistency · 90 / 100
    Check that role, task, format, exclusions, and examples do not contradict each other.
  • Practicality · 90 / 100
    Add runnable steps, examples, or verification cues instead of abstract advice.
  • Specificity · 70 / 100
    Anchor wording in the user's audience, domain nouns, and target outcome.
+
+
+

RTF to skill mapping

+
  • Role
    Use a conversational role that asks only high-leverage questions and remembers the user's goal.
  • Task
    Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.
  • Format
    Return concise prompts, decision points, and reviewer-visible assumptions.
+
+
+ +
+
+

Reference coach

+

obra/superpowers

Borrow now

  • Borrow the way it turns a messy workflow into a repeatable operating path.
  • Borrow the clear execution entrypoints and command structure.

Avoid

  • Do not import process overhead that only exists for that project's scale.

affaan-m/ECC

Borrow now

  • Borrow the way it turns a messy workflow into a repeatable operating path.
  • Borrow the clear execution entrypoints and command structure.

Avoid

  • Do not import process overhead that only exists for that project's scale.

multica-ai/andrej-karpathy-skills

Borrow now

  • Borrow explicit validation and quality gates that make iteration safer.
  • Borrow the way it separates explanation, examples, and reusable structure.

Avoid

  • Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.
+
+
+

Decide before you deepen

+
    +
  • Choose one pattern to borrow on purpose, not three at once.
    • +
  • State one thing this skill will not inherit from the benchmark objects.
    • +
  • Only deepen the package after that choice is visible in the boundary or execution flow.
    • +
+
+
+ +
+
+

Reference synthesis

+

Official workflow product ergonomics

Borrow now

  • Borrow a first-time operator flow that explains itself before it asks for more structure.

Avoid

  • Do not mimic product polish that adds UI bulk without improving clarity.

Hypothesis-test-learn loop

Borrow now

  • Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.

Avoid

  • Do not create experimental overhead that exceeds the skill's real risk tier.

Boundary-first design

Borrow now

  • Borrow the discipline of defining what the skill should not own before growing the package.

Avoid

  • Do not expand execution assets until route boundaries stay clean.
+
+
+

Borrow now

+
  • Borrow a first-time operator flow that explains itself before it asks for more structure.
  • Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.
  • Borrow the discipline of defining what the skill should not own before growing the package.
  • Borrow the way it turns a messy workflow into a repeatable operating path.
+

Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.

+
+
+ +
+

Top three next moves

+

Borrow one proven pattern on purpose

You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.

  • Read the strongest pattern from obra/superpowers.
  • Decide whether to borrow method, structure, execution, or portability, but only one of them first.
  • Record what you will not borrow so the package stays light.
Unlocks: A cleaner package shape with less accidental over-design.

Harden portability semantics

The skill already signals reuse across environments, so contract clarity matters early.

  • Confirm activation mode, execution context, and trust assumptions.
  • Add or review degradation strategy for non-native targets.
  • Package the skill once to verify adapter expectations.
Unlocks: Safer cross-environment reuse with less target drift.

Create an iteration evidence loop

The package should show what changed and why after the first draft.

  • Generate the HTML skill report and keep it aligned with the package.
  • Record reference scan choices and non-goals.
  • Capture the next iteration choice explicitly before adding more files.
Unlocks: A clearer path for the next author or reviewer.
+
+ +
+
+

Recent feedback

+
  • No lightweight feedback captured yet. Use `yao.py feedback` to record quick review notes.
+
+
+

Promotion status

+

Promote: 0

Keep current: 3

Blocked: 0

+
+
+ +
+
+

Package map

+
  • SKILL.md — Skill entrypoint
  • README.md — Human-readable usage guide
  • agents/interface.yaml — Neutral interface metadata
  • manifest.json — Lifecycle and portability metadata
  • references — Extended guidance and reusable notes
  • scripts — Deterministic helpers or local tooling
  • evals — Trigger and quality checks
  • reports — Generated evidence and overview artifacts
+
+
+

First-pass review frame

+
    +
  • Does the trigger stay narrow enough for the intended job?
    • +
  • Does the archetype match the real reuse level?
    • +
  • Are we adding structure faster than we are adding reliability?
    • +
  • Should the next step be trigger tightening, execution assets, or portability hardening?
    • +
+
+
+ +
+
+

Authoring discipline

+
    +
  • Name unresolved assumptions before deepening the package.
    • +
  • Keep the package no larger than the recurring job requires.
    • +
  • Touch only files that directly support the requested change.
    • +
  • Tie every meaningful new artifact to a check or reviewer note.
    • +
+
+
+

Reviewer guardrails

+
    +
  • Block speculative features that are not backed by real workflow variation.
    • +
  • Move unverifiable ideas into next-step candidates instead of baseline structure.
    • +
  • Reject decorative folders, reports, or governance that do not reduce risk.
    • +
  • Ask for one high-leverage clarification when job, output, or exclusion is still fuzzy.
    • +
+
+
+
+ + diff --git a/reports/review-viewer.json b/reports/review-viewer.json new file mode 100644 index 00000000..93c362a8 --- /dev/null +++ b/reports/review-viewer.json @@ -0,0 +1,13063 @@ +{ + "overview": { + "skill_summary": { + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "display_name": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "maturity": "governed", + "updated_at": "2026-03-31", + "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", + "audience": "Skill 作者、复用团队和后续 reviewer。", + "deliverables": [ + "SKILL.md", + "agents/interface.yaml", + "reports/skill-overview.html" + ], + "flow": [ + "输入材料", + "Skill 包体", + "可复用能力" + ] + }, + "scorecard": { + "completeness_score": { + "label": "完整度", + "score": 100, + "reasons": [ + "SKILL.md 已存在，是 Skill 的入口。", + "README.md 已存在，便于人工阅读。", + "agents/interface.yaml 已存在，便于跨平台适配。", + "manifest.json 已存在，生命周期信息可追踪。", + "reports/ 已存在，生成证据可以随包体迁移。" + ] + }, + "trigger_score": { + "label": "触发清晰", + "score": 100, + "reasons": [ + "frontmatter description 已存在，具备基础路由面。", + "description 有足够长度说明任务边界。", + "description 已包含使用场景或排除边界信号。", + "evals/ 已存在，可承载触发样例或质量检查。", + "intent-confidence 报告已生成，可辅助判断触发稳定性。" + ] + }, + "evidence_score": { + "label": "证据充分", + "score": 75, + "reasons": [ + "已生成 6 / 8 类报告证据。", + "intent-confidence.json 已存在。", + "reference-synthesis.json 已存在。", + "artifact-design-profile.json 已存在。", + "证据不足：缺少 intent-dialogue.json, output-risk-profile.json。" + ] + }, + "maintainability_score": { + "label": "可维护性", + "score": 100, + "reasons": [ + "SKILL.md 约 362 个词/字。", + "入口文件保持克制，可维护性较好。", + "references/ 已承载扩展指导。", + "scripts/ 已承载确定性逻辑。", + "evals/ 已承载可迁移检查。" + ] + }, + "portability_score": { + "label": "可迁移性", + "score": 100, + "reasons": [ + "agents/interface.yaml 已存在。", + "manifest.json 已存在。", + "目标平台或 adapter target 已声明。", + "入口文件未发现明显私有绝对路径。" + ] + }, + "context_cost": { + "label": "上下文成本", + "score": 42, + "reasons": [ + "入口约 362 个词/字，references 约 10018 个词/字。", + "分数越高代表上下文成本越低。", + "上下文成本偏高，建议压缩入口或拆分 references。" + ] + } + }, + "capability_profile": { + "archetype": "governed", + "task_family": "Dialogue interaction", + "maturity": "governed", + "trigger_strength": "手动触发 + description 路由", + "reuse_scope": "跨平台", + "matrix": { + "execution_certainty": 72, + "knowledge_density": 80 + } + }, + "principle_model": { + "nodes": [ + { + "title": "意图澄清", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "layers": [ + "入口层", + "参考层", + "脚本层", + "评估层", + "报告层" + ] + }, + "contract_boundary": { + "trigger": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ], + "should_trigger": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。" + ], + "should_not_trigger": [ + "只需要一次性回答、没有复用价值的临时请求。", + "要求直接执行相邻任务，而不是沉淀或使用这个 Skill。", + "缺少必要事实且用户不允许澄清的场景。" + ], + "boundary_cards": [ + { + "label": "Owned", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "label": "Adjacent", + "body": "相邻任务需要先确认是否应转为独立 Skill。" + }, + { + "label": "Excluded", + "body": "不替代人工事实核查，也不静默扩大职责。" + } + ] + }, + "quality_review": { + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已打包 agents/interface.yaml，便于后续做跨平台适配。", + "长指导被拆到 references 中，入口文件可以保持轻量。", + "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。", + "包内包含可随 Skill 迁移的质量门禁或触发检查。" + ], + "gaps": [ + "上下文成本需要补强：入口约 362 个词/字，references 约 10018 个词/字。" + ], + "recommendations": [ + "先改触发边界，再扩展工作流。", + "只把重复且稳定的步骤沉淀为脚本。", + "每次升级后重新生成报告并检查分数原因。" + ], + "artifact_design": { + "design_system": "content-led editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 82.0, + "highlights": [ + "Primary prompt task family: Dialogue interaction.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 80/100." + ] + }, + "system_model": { + "stability": { + "score": 78, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (78/100).", + "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + } + }, + "risk_governance": { + "risks": [ + { + "name": "误触发风险", + "impact": 3, + "probability": 1, + "signal": "frontmatter description 已存在，具备基础路由面。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "输出漂移风险", + "impact": 2, + "probability": 2, + "signal": "已生成 6 / 8 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "证据不足风险", + "impact": 3, + "probability": 2, + "signal": "已生成 6 / 8 类报告证据。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "包体膨胀风险", + "impact": 2, + "probability": 1, + "signal": "SKILL.md 约 362 个词/字。", + "response": "先补证据和边界，再增加包体复杂度。" + }, + { + "name": "跨平台迁移风险", + "impact": 3, + "probability": 1, + "signal": "agents/interface.yaml 已存在。", + "response": "先补证据和边界，再增加包体复杂度。" + } + ], + "risk_families": [], + "human_judgment_boundary": [ + "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", + "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", + "Do not silently broaden the skill into adjacent jobs just because the examples are nearby." + ] + }, + "package_assets": { + "entries": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 23 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 65 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 25 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 118 + } + ], + "file_count": 235, + "folder_count": 4, + "distribution": [ + { + "label": "SKILL.md", + "value": 1 + }, + { + "label": "README.md", + "value": 1 + }, + { + "label": "agents/interface.yaml", + "value": 1 + }, + { + "label": "manifest.json", + "value": 1 + }, + { + "label": "references", + "value": 23 + }, + { + "label": "scripts", + "value": 65 + }, + { + "label": "evals", + "value": 25 + }, + { + "label": "reports", + "value": 118 + } + ] + }, + "iteration_roadmap": { + "items": [ + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + }, + { + "title": "Create an iteration evidence loop", + "why": "The package should show what changed and why after the first draft.", + "actions": [ + "Generate the HTML skill report and keep it aligned with the package.", + "Record reference scan choices and non-goals.", + "Capture the next iteration choice explicitly before adding more files." + ], + "unlocks": "A clearer path for the next author or reviewer." + } + ] + }, + "report_contract": { + "schema_version": "2.0", + "html_report": "reports/skill-overview.html", + "language": "zh-CN", + "default_language": "zh-CN", + "languages": [ + "zh-CN", + "en" + ], + "layout": "kami-white-audit-v2", + "nav_labels": [ + "技能概述", + "总览指标", + "能力画像", + "原理结构", + "契约边界", + "质量评估", + "风险治理", + "包体资产", + "迭代路线" + ], + "nav_labels_en": [ + "Overview", + "Metrics", + "Profile", + "Principle", + "Contract", + "Quality", + "Risk", + "Assets", + "Roadmap" + ] + }, + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "display_name": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "logic_steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter." + ], + "usage_steps": [ + "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.", + "Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + ], + "package_map": [ + { + "path": "SKILL.md", + "label": "Skill entrypoint", + "kind": "file", + "file_count": 1 + }, + { + "path": "README.md", + "label": "Human-readable usage guide", + "kind": "file", + "file_count": 1 + }, + { + "path": "agents/interface.yaml", + "label": "Neutral interface metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "manifest.json", + "label": "Lifecycle and portability metadata", + "kind": "file", + "file_count": 1 + }, + { + "path": "references", + "label": "Extended guidance and reusable notes", + "kind": "folder", + "file_count": 23 + }, + { + "path": "scripts", + "label": "Deterministic helpers or local tooling", + "kind": "folder", + "file_count": 65 + }, + { + "path": "evals", + "label": "Trigger and quality checks", + "kind": "folder", + "file_count": 25 + }, + { + "path": "reports", + "label": "Generated evidence and overview artifacts", + "kind": "folder", + "file_count": 118 + } + ], + "strengths": [ + "触发面保持精简，并锚定在 frontmatter description。", + "已打包 agents/interface.yaml，便于后续做跨平台适配。", + "长指导被拆到 references 中，入口文件可以保持轻量。", + "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。", + "包内包含可随 Skill 迁移的质量门禁或触发检查。" + ], + "scenarios": [ + "把重复流程沉淀为可复用的 agent skill。", + "把分散提示词、对话记录或操作规范整理为稳定能力。", + "团队复用前，需要明确触发边界、质量证据和维护责任。", + "用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals." + ], + "trigger_contract": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "activation": "manual", + "execution": "inline", + "shell": "bash" + }, + "io_contract": { + "inputs": [ + "用户提供的工作流、提示词、文档、记录或散乱笔记", + "期望沉淀的复用场景、排除项、约束和质量标准" + ], + "outputs": [ + "可路由的 SKILL.md", + "agents/interface.yaml 元数据", + "必要的 references、scripts、evals、reports 证据", + "结构化 Skill 目录，共 8 类关键资产。" + ] + }, + "principles": [ + { + "title": "意图澄清", + "body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + }, + { + "title": "边界路由", + "body": "用 frontmatter description 决定是否触发，并写明相邻非目标。" + }, + { + "title": "资产分层", + "body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。" + }, + { + "title": "证据回路", + "body": "Ask only the highest-leverage clarification before adding package weight." + }, + { + "title": "漂移观察", + "body": "Add near-neighbor exclusions and route evals before expanding workflow steps." + }, + { + "title": "杠杆升级", + "body": "Ask one focused question and update intent context before adding assets." + } + ], + "roadmap": [ + { + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift." + }, + { + "title": "Create an iteration evidence loop", + "why": "The package should show what changed and why after the first draft.", + "actions": [ + "Generate the HTML skill report and keep it aligned with the package.", + "Record reference scan choices and non-goals.", + "Capture the next iteration choice explicitly before adding more files." + ], + "unlocks": "A clearer path for the next author or reviewer." + } + ], + "cards": [], + "introduction": [ + "这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。", + "先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。", + "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" + ], + "benchmark_highlights": [], + "synthesis_highlights": [ + "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package." + ], + "artifact_design": { + "design_system": "content-led editorial", + "highlights": [ + "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers." + ] + }, + "prompt_quality": { + "overall_quality_score": 82.0, + "highlights": [ + "Primary prompt task family: Dialogue interaction.", + "Complexity: complex — multiple inputs, constraints, or task families require tradeoff handling", + "Completeness: 80/100.", + "Clarity: 80/100." + ] + }, + "system_model": { + "stability": { + "score": 78, + "band": "stable-first-pass" + }, + "highlights": [ + "Stability: stable-first-pass (78/100).", + "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", + "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." + ] + }, + "metadata": { + "canonical_format": "agent-skills", + "targets": [ + "openai", + "claude", + "generic" + ], + "maturity_tier": "governed", + "skill_archetype": "governed", + "updated_at": "2026-03-31" + } + }, + "intent_confidence": { + "score": 30, + "band": "low", + "gate_passed": false, + "strengths": [ + "The recurring job is concrete enough to anchor the package." + ], + "gaps": [ + { + "key": "real_inputs", + "label": "Real inputs are missing", + "reason": "Without real inputs, it is hard to choose assets, scripts, or examples.", + "severity": "high" + }, + { + "key": "primary_output", + "label": "Primary output is missing", + "reason": "The package does not yet know what it must hand back.", + "severity": "high" + }, + { + "key": "exclusions", + "label": "Near-neighbor exclusions are missing", + "reason": "The route may blur into nearby requests without an exclusion list.", + "severity": "high" + }, + { + "key": "constraints", + "label": "Constraints are missing", + "reason": "The package does not yet know which tradeoffs matter most.", + "severity": "high" + }, + { + "key": "standards", + "label": "Quality bar is implied, not explicit", + "reason": "The first evaluation target is still underspecified.", + "severity": "medium" + } + ], + "follow_up_questions": [ + { + "slot": "real_inputs", + "question": "What material will people actually hand to this skill in practice?", + "why": "Real input shape decides whether references, scripts, or examples are needed.", + "list": true, + "label": "Real inputs are missing", + "severity": "high" + }, + { + "slot": "primary_output", + "question": "What finished hand-back should this skill return so the next person can keep moving?", + "why": "The output is the anchor for package design and review.", + "list": false, + "label": "Primary output is missing", + "severity": "high" + }, + { + "slot": "exclusions", + "question": "What nearby requests should this skill clearly leave out so the boundary stays clean?", + "why": "Exclusions are the fastest route to better trigger quality.", + "list": true, + "label": "Near-neighbor exclusions are missing", + "severity": "high" + } + ], + "anchor_sentence": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "recommended_action": "Pause before deep authoring and close the highest-leverage gaps first.", + "context": { + "job": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "real_inputs": [], + "primary_output": "", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "exclusions": [], + "constraints": [], + "standards": [], + "correction": "", + "user_references": [] + } + }, + "intent": { + "skill_name": "yao-meta-skill", + "title": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "focus": "portability-and-contract", + "opening_frame": "Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that.", + "reference_note": "If you already have examples you admire, bring them in. We will learn the pattern, not copy the source.", + "conversation_path": [ + "Start with the user's own words, not package vocabulary.", + "Reflect the job, output, and non-goals back in one clean sentence.", + "Only then offer a tiny scaffold if it would help the user move faster." + ], + "opening_styles": [ + { + "label": "温柔陪伴型", + "best_when": "用户想法还散、还在试探，或者需要先被接住。", + "message": "我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。" + }, + { + "label": "专业教练型", + "best_when": "用户目标比较明确，希望被高效带着走。", + "message": "我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。" + }, + { + "label": "共创伙伴型", + "best_when": "用户已经有一些想法，希望一起打磨，不想被填表。", + "message": "我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。" + } + ], + "optional_scaffold": [ + "The repeated job it should reliably handle", + "The real inputs people will hand to it", + "The useful output it should hand back", + "What it should clearly refuse" + ], + "questions": [ + { + "question": "If this skill worked beautifully, what recurring job would it reliably handle for the user every time?", + "why": "This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape." + }, + { + "question": "When someone reaches for this skill in the real world, what materials will they actually hand to it?", + "why": "Input shape decides whether references, scripts, or templates are needed." + }, + { + "question": "What finished output should it hand back so the user can immediately keep moving?", + "why": "Outputs should drive the package structure before extra guidance is added." + }, + { + "question": "Which nearby requests should this skill politely refuse so the boundary stays clean?", + "why": "The exclusion list is the fastest route to better trigger quality." + }, + { + "question": "What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?", + "why": "Constraints decide how much structure, packaging, and review the skill actually needs." + }, + { + "question": "Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example?", + "why": "A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material." + }, + { + "question": "Which environments or clients must be able to consume this skill?", + "why": "This sets the minimum metadata and degradation strategy." + } + ], + "output": { + "capability_sentence": "Yao Meta Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily.", + "required_capture": [ + "recurring job", + "real inputs", + "required outputs", + "exclusions", + "constraints", + "reference preferences", + "first evaluation target" + ], + "recommended_first_gate": "portability and contract" + } + }, + "reference": { + "skill_name": "yao-meta-skill", + "title": "Yao Meta Skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "scan_focus": [ + { + "label": "Evaluation pattern", + "reason": "This skill already carries eval assets, so benchmark how top examples define trigger boundaries and quality gates." + }, + { + "label": "Execution pattern", + "reason": "There is deterministic logic in scripts, so compare how strong references separate prose from executable steps." + }, + { + "label": "Portability pattern", + "reason": "The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source." + }, + { + "label": "Method pattern", + "reason": "Use the core job description as the anchor for comparison: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse." + } + ], + "external_references": [], + "user_references": [], + "local_constraints": [], + "borrow_plan": [ + "External benchmark first: let high-quality public references define the upper bound for method, structure, execution, or portability.", + "User references second: use them to understand taste, standards, and directional preferences without copying source phrasing.", + "Local fit third: use local assets only to detect naming conflicts, private dependencies, or compatibility constraints.", + "Borrow patterns, not prose: extract loops, boundaries, metadata, and operator flow without copying source-specific language.", + "Keep the package light: reject any borrowed pattern that increases context cost faster than it increases reliability." + ], + "priority_rule": "External benchmark objects set the pattern ceiling. User references refine taste and standards. Local files only calibrate fit, risk, and compatibility.", + "non_goals": [ + "Do not copy source prose or branding into the new skill.", + "Do not import gates that cost more context than they save.", + "Do not use benchmark scanning to justify scope creep.", + "Do not let local historical habits outrank stronger public benchmarks." + ] + }, + "iteration": { + "summary": { + "skill_name": "yao-meta-skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "maturity_tier": "governed", + "selection_rule": "Pick the three smallest next steps that increase reliability more than they increase context cost.", + "recommended_now": "Borrow one proven pattern on purpose", + "recommended_now_why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "defer_for_now": "Create an iteration evidence loop" + }, + "directions": [ + { + "priority": 2, + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design.", + "do_now": "Do this first.", + "wait_on": "Wait to add broader structure until this move clearly improves reliability." + }, + { + "priority": 5, + "title": "Harden portability semantics", + "why": "The skill already signals reuse across environments, so contract clarity matters early.", + "actions": [ + "Confirm activation mode, execution context, and trust assumptions.", + "Add or review degradation strategy for non-native targets.", + "Package the skill once to verify adapter expectations." + ], + "unlocks": "Safer cross-environment reuse with less target drift.", + "do_now": "Do this after the first move lands cleanly.", + "wait_on": "Wait until the package has evidence that this extra structure is justified." + }, + { + "priority": 12, + "title": "Create an iteration evidence loop", + "why": "The package should show what changed and why after the first draft.", + "actions": [ + "Generate the HTML skill report and keep it aligned with the package.", + "Record reference scan choices and non-goals.", + "Capture the next iteration choice explicitly before adding more files." + ], + "unlocks": "A clearer path for the next author or reviewer.", + "do_now": "Do this after the first move lands cleanly.", + "wait_on": "Wait until the package has evidence that this extra structure is justified." + } + ] + }, + "feedback": {}, + "baseline": { + "summary": { + "target_count": 3, + "baseline_total_errors": 8, + "current_total_errors": 5, + "winner_total_errors": 5, + "winner_vs_baseline_gain": 3, + "winner_vs_current_gain": 0 + }, + "comparisons": [ + { + "label": "root", + "winner_label": "Current", + "baseline": { + "tokens": 8, + "dev_errors": 1, + "holdout_errors": 0, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "current": { + "tokens": 65, + "dev_errors": 0, + "holdout_errors": 0, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "winner": { + "tokens": 65, + "dev_errors": 0, + "holdout_errors": 0, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "delta": { + "current_vs_baseline": 1, + "winner_vs_baseline": 1, + "winner_vs_current": 0 + } + }, + { + "label": "team-frontend-review", + "winner_label": "Current", + "baseline": { + "tokens": 52, + "dev_errors": 3, + "holdout_errors": 0, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "current": { + "tokens": 50, + "dev_errors": 3, + "holdout_errors": 0, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "winner": { + "tokens": 50, + "dev_errors": 3, + "holdout_errors": 0, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "delta": { + "current_vs_baseline": 0, + "winner_vs_baseline": 0, + "winner_vs_current": 0 + } + }, + { + "label": "governed-incident-command", + "winner_label": "Current", + "baseline": { + "tokens": 93, + "dev_errors": 1, + "holdout_errors": 2, + "blind_errors": 1, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "current": { + "tokens": 37, + "dev_errors": 1, + "holdout_errors": 1, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "winner": { + "tokens": 37, + "dev_errors": 1, + "holdout_errors": 1, + "blind_errors": 0, + "judge_blind_errors": 0, + "adversarial_errors": 0 + }, + "delta": { + "current_vs_baseline": 2, + "winner_vs_baseline": 2, + "winner_vs_current": 0 + } + } + ] + }, + "compare": { + "current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "current_candidate": { + "id": "current", + "label": "Current", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "strategy": "current", + "estimated_tokens": 65, + "dev": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "selection_key": [ + 0, + 0, + -1.0, + -1.0, + -1.0, + -1.0, + 65 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + "baseline": { + "description": "Create and improve agent skills.", + "estimated_tokens": 8, + "dev": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 1, + "precision": 1.0, + "recall": 0.875, + "bucket_stats": { + "should_trigger": { + "total": 8, + "passed": 7, + "pass_rate": 0.875 + }, + "should_not_trigger": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + } + }, + "family_stats": { + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 2, + "passed": 1, + "false_positives": 0, + "false_negatives": 1, + "pass_rate": 0.5 + }, + "complex_multi_asset": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_not_package": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "partial_scaffold_not_full_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [ + { + "bucket": "should_trigger", + "family": "iterate_existing_skill", + "prompt": "Add trigger evals to this skill before sharing it with the team.", + "score": 0.328, + "reason": "false_negative", + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill" + ], + "matched_negative_concepts": [] + } + ], + "results": { + "should_trigger": [ + { + "prompt": "Convert this operations checklist into a reusable skill.", + "family": "workflow_to_skill", + "score": 0.387, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.377, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "checklist", + "operations checklist" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Add trigger evals to this skill before sharing it with the team.", + "family": "iterate_existing_skill", + "score": 0.328, + "predicted_trigger": false, + "expected_trigger": true, + "passed": false, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill" + ], + "extra_positive_concepts": [ + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.302, + "support_score": 0.04, + "lexical_support": 0.0, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "trigger evals" + ], + "iterate_existing_skill": [ + "this skill", + "sharing it with the team", + "before sharing it with the team" + ], + "team_operationalize": [ + "team", + "sharing it with the team" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", + "family": "complex_multi_asset", + "score": 0.844, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0, + "lexical_support": 0.042, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package", + "turn all of that into one reusable skill package" + ], + "eval_optimize": [ + "evals", + "packaging checks" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "runbook", + "process", + "prompt history" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", + "family": "complex_multi_asset", + "score": 0.408, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.377, + "support_score": 0.06, + "lexical_support": 0.067, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "production-ready" + ], + "transform_workflow": [ + "workflow", + "workflow fragments" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", + "family": "iterate_existing_skill", + "score": 0.527, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package" + ], + "extra_positive_concepts": [ + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.491, + "support_score": 0.04, + "lexical_support": 0.071, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "distribution" + ], + "team_operationalize": [ + "team", + "distribution" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", + "family": "paraphrase_trigger", + "score": 0.403, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.377, + "support_score": 0.04, + "lexical_support": 0.071, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "team library", + "maintained" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", + "family": "paraphrase_trigger", + "score": 0.856, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0.04, + "lexical_support": 0.067, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this workflow" + ], + "eval_optimize": [ + "validate", + "route boundary" + ], + "reuse_package": [ + "package", + "distribution", + "shared capability" + ], + "team_operationalize": [ + "shared", + "distribution" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", + "family": "long_context_trigger", + "score": 0.869, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset", + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0.1, + "lexical_support": 0.038, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill", + "skill package" + ], + "eval_optimize": [ + "validation" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ], + "transform_workflow": [ + "rough notes", + "transcript", + "onboarding flow" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Give me ideas for improving our process.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0.24, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "process" + ] + }, + "negative": { + "brainstorm_only": [ + "ideas" + ] + } + } + } + }, + { + "prompt": "Review this note and tell me what it means in plain English.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "tell me what it means", + "plain english" + ] + } + } + } + }, + { + "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "semantic_coverage": 0.189, + "support_score": 0.06, + "lexical_support": 0.067, + "negative_penalty": 0.66, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm", + "improve packaging" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Translate these notes into French and keep the structure exactly as-is.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "translate_only" + ], + "exclusive_negative_concepts": [ + "translate_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "translate_only": [ + "translate", + "into french" + ] + } + } + } + }, + { + "prompt": "Summarize this workflow and list the main points only.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.111, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "summarize_only": [ + "summarize" + ] + } + } + } + }, + { + "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.043, + "negative_penalty": 0.68, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "explain_only": [ + "explain it", + "in simpler language" + ], + "no_build_directive": [ + "do not convert it into a reusable capability", + "keep it as documentation only" + ] + } + } + } + }, + { + "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", + "family": "long_context_summary_only", + "score": 0.257, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.698, + "support_score": 0, + "lexical_support": 0.056, + "negative_penalty": 0.3, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "summarize_only": [ + "summary", + "recap" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Review this process note and explain it, no packaging needed.", + "family": "explain_not_package", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.377, + "support_score": 0, + "lexical_support": 0.1, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "explain_only": [ + "explain it" + ], + "no_build_directive": [ + "no packaging needed" + ] + } + } + } + }, + { + "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.071, + "negative_penalty": 0.42, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "no_build_directive": [ + "stop short of making a full skill" + ] + } + } + } + }, + { + "prompt": "Package this explanation as a document, not as an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.1, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document" + ], + "no_build_directive": [ + "not as an agent skill" + ] + } + } + } + }, + { + "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "iterate_existing_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.34, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "iterate_existing_skill": [ + "this skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.071, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "document_only": [ + "document", + "reusable-looking document" + ], + "no_build_directive": [ + "without turning them into an agent skill" + ] + } + } + } + }, + { + "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.048, + "negative_penalty": 0.82, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document", + "template only" + ], + "future_outline": [ + "not now", + "later but not now" + ], + "no_build_directive": [ + "not now", + "later but not now" + ] + } + } + } + }, + { + "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.053, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "for later", + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + } + ] + } + }, + "holdout": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 5, + "passed": 5, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + } + }, + "family_stats": { + "complex_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "family": "complex_multi_asset", + "score": 0.416, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset", + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.377, + "support_score": 0.1, + "lexical_support": 0.053, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable" + ], + "team_operationalize": [ + "internal" + ], + "transform_workflow": [ + "process", + "internal process note" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", + "family": "iterate_existing_skill", + "score": 0.699, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.679, + "support_score": 0.04, + "lexical_support": 0.056, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "near-neighbor", + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "package", + "distribution" + ], + "team_operationalize": [ + "distribution" + ], + "transform_workflow": [ + "existing skill draft" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", + "family": "workflow_to_skill", + "score": 0.546, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.528, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "library entry" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", + "family": "paraphrase_trigger", + "score": 0.861, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0.06, + "lexical_support": 0.062, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "packaging checks" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "quarterly release routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", + "family": "long_context_trigger", + "score": 0.719, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "team_operationalize" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.698, + "support_score": 0.04, + "lexical_support": 0.08, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "turn this playbook into an agent capability package" + ], + "reuse_package": [ + "package", + "team library" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "prompt history", + "transcript", + "release notes" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "semantic_coverage": 0.377, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0.72, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "no_build_directive": [ + "not a reusable skill" + ], + "summarize_only": [ + "summary", + "only want a summary" + ] + } + } + } + }, + { + "prompt": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.377, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "explain_only": [ + "explain this", + "in simpler language" + ], + "no_build_directive": [ + "do not package it" + ] + } + } + } + }, + { + "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.111, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document" + ], + "no_build_directive": [ + "not for agent execution" + ] + } + } + } + }, + { + "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "semantic_coverage": 0.698, + "support_score": 0, + "lexical_support": 0.062, + "negative_penalty": 0.77, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "no_build_directive": [ + "no skill package", + "keep it as documentation only" + ], + "translate_only": [ + "translate", + "into russian" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "semantic_coverage": 0.321, + "support_score": 0, + "lexical_support": 0.0, + "negative_penalty": 0.46, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ] + }, + "negative": { + "brainstorm_only": [ + "shape an idea" + ], + "future_outline": [ + "before we decide whether to build" + ] + } + } + } + }, + { + "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.071, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill", + "outline for a possible future skill" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.077, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document", + "document package", + "wiki" + ], + "no_build_directive": [ + "not an agent skill" + ] + } + } + } + }, + { + "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.698, + "support_score": 0, + "lexical_support": 0.125, + "negative_penalty": 0.6, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "workflow", + "checklist" + ] + }, + "negative": { + "document_only": [ + "template only" + ], + "no_build_directive": [ + "no skill package", + "discussion only" + ] + } + } + } + } + ] + } + } + }, + "winner": { + "id": "current", + "label": "Current", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "strategy": "current", + "estimated_tokens": 65, + "dev": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "selection_key": [ + 0, + 0, + -1.0, + -1.0, + -1.0, + -1.0, + 65 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + "winner_dev_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 8, + "passed": 8, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + } + }, + "family_stats": { + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "complex_multi_asset": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_not_package": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "partial_scaffold_not_full_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Convert this operations checklist into a reusable skill.", + "family": "workflow_to_skill", + "score": 0.415, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "checklist", + "operations checklist" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Add trigger evals to this skill before sharing it with the team.", + "family": "iterate_existing_skill", + "score": 0.394, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.327, + "support_score": 0, + "lexical_support": 0.417, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "trigger evals" + ], + "iterate_existing_skill": [ + "this skill", + "sharing it with the team", + "before sharing it with the team" + ], + "team_operationalize": [ + "team", + "sharing it with the team" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", + "family": "complex_multi_asset", + "score": 0.843, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package", + "turn all of that into one reusable skill package" + ], + "eval_optimize": [ + "evals", + "packaging checks" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "runbook", + "process", + "prompt history" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", + "family": "complex_multi_asset", + "score": 0.422, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.4, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "production-ready" + ], + "transform_workflow": [ + "workflow", + "workflow fragments" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", + "family": "iterate_existing_skill", + "score": 0.557, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "distribution" + ], + "team_operationalize": [ + "team", + "distribution" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", + "family": "paraphrase_trigger", + "score": 0.474, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.4, + "support_score": 0, + "lexical_support": 0.571, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "reuse_package": [ + "team library", + "maintained" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", + "family": "paraphrase_trigger", + "score": 0.868, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.267, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this workflow" + ], + "eval_optimize": [ + "validate", + "route boundary" + ], + "reuse_package": [ + "package", + "distribution", + "shared capability" + ], + "team_operationalize": [ + "shared", + "distribution" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", + "family": "long_context_trigger", + "score": 0.889, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0.06, + "lexical_support": 0.346, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill", + "skill package" + ], + "eval_optimize": [ + "validation" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ], + "transform_workflow": [ + "rough notes", + "transcript", + "onboarding flow" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Give me ideas for improving our process.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.24, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "process" + ] + }, + "negative": { + "brainstorm_only": [ + "ideas" + ] + } + } + } + }, + { + "prompt": "Review this note and tell me what it means in plain English.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "tell me what it means", + "plain english" + ] + } + } + } + }, + { + "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0.06, + "lexical_support": 0.2, + "negative_penalty": 0.66, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm", + "improve packaging" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Translate these notes into French and keep the structure exactly as-is.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "translate_only" + ], + "exclusive_negative_concepts": [ + "translate_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "translate_only": [ + "translate", + "into french" + ] + } + } + } + }, + { + "prompt": "Summarize this workflow and list the main points only.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.111, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "summarize_only": [ + "summarize" + ] + } + } + } + }, + { + "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.174, + "negative_penalty": 0.68, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "explain_only": [ + "explain it", + "in simpler language" + ], + "no_build_directive": [ + "do not convert it into a reusable capability", + "keep it as documentation only" + ] + } + } + } + }, + { + "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", + "family": "long_context_summary_only", + "score": 0.256, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.333, + "negative_penalty": 0.3, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "summarize_only": [ + "summary", + "recap" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Review this process note and explain it, no packaging needed.", + "family": "explain_not_package", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "explain_only": [ + "explain it" + ], + "no_build_directive": [ + "no packaging needed" + ] + } + } + } + }, + { + "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0.42, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "no_build_directive": [ + "stop short of making a full skill" + ] + } + } + } + }, + { + "prompt": "Package this explanation as a document, not as an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document" + ], + "no_build_directive": [ + "not as an agent skill" + ] + } + } + } + }, + { + "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "iterate_existing_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.327, + "support_score": 0, + "lexical_support": 0.267, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "iterate_existing_skill": [ + "this skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "document_only": [ + "document", + "reusable-looking document" + ], + "no_build_directive": [ + "without turning them into an agent skill" + ] + } + } + } + }, + { + "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.238, + "negative_penalty": 0.82, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document", + "template only" + ], + "future_outline": [ + "not now", + "later but not now" + ], + "no_build_directive": [ + "not now", + "later but not now" + ] + } + } + } + }, + { + "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.263, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "for later", + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + } + ] + } + }, + "winner_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 5, + "passed": 5, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + } + }, + "family_stats": { + "complex_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "family": "complex_multi_asset", + "score": 0.468, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.4, + "support_score": 0.06, + "lexical_support": 0.316, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable" + ], + "team_operationalize": [ + "internal" + ], + "transform_workflow": [ + "process", + "internal process note" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", + "family": "iterate_existing_skill", + "score": 0.727, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.691, + "support_score": 0, + "lexical_support": 0.389, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "near-neighbor", + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "package", + "distribution" + ], + "team_operationalize": [ + "distribution" + ], + "transform_workflow": [ + "existing skill draft" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", + "family": "workflow_to_skill", + "score": 0.568, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "library entry" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", + "family": "paraphrase_trigger", + "score": 0.858, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0.06, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "packaging checks" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "quarterly release routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", + "family": "long_context_trigger", + "score": 0.744, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "turn this playbook into an agent capability package" + ], + "reuse_package": [ + "package", + "team library" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "prompt history", + "transcript", + "release notes" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.72, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "no_build_directive": [ + "not a reusable skill" + ], + "summarize_only": [ + "summary", + "only want a summary" + ] + } + } + } + }, + { + "prompt": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.091, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "explain_only": [ + "explain this", + "in simpler language" + ], + "no_build_directive": [ + "do not package it" + ] + } + } + } + }, + { + "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.222, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document" + ], + "no_build_directive": [ + "not for agent execution" + ] + } + } + } + }, + { + "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.77, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "no_build_directive": [ + "no skill package", + "keep it as documentation only" + ], + "translate_only": [ + "translate", + "into russian" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "semantic_coverage": 0.309, + "support_score": 0, + "lexical_support": 0.308, + "negative_penalty": 0.46, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ] + }, + "negative": { + "brainstorm_only": [ + "shape an idea" + ], + "future_outline": [ + "before we decide whether to build" + ] + } + } + } + }, + { + "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill", + "outline for a possible future skill" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.615, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document", + "document package", + "wiki" + ], + "no_build_directive": [ + "not an agent skill" + ] + } + } + } + }, + { + "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.438, + "negative_penalty": 0.6, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "workflow", + "checklist" + ] + }, + "negative": { + "document_only": [ + "template only" + ], + "no_build_directive": [ + "no skill package", + "discussion only" + ] + } + } + } + } + ] + } + }, + "current_dev_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 8, + "passed": 8, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 7, + "passed": 7, + "pass_rate": 1.0 + } + }, + "family_stats": { + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "complex_multi_asset": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "brainstorm_only": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_not_package": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "partial_scaffold_not_full_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Convert this operations checklist into a reusable skill.", + "family": "workflow_to_skill", + "score": 0.415, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "checklist", + "operations checklist" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Add trigger evals to this skill before sharing it with the team.", + "family": "iterate_existing_skill", + "score": 0.394, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.327, + "support_score": 0, + "lexical_support": 0.417, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "trigger evals" + ], + "iterate_existing_skill": [ + "this skill", + "sharing it with the team", + "before sharing it with the team" + ], + "team_operationalize": [ + "team", + "sharing it with the team" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", + "family": "complex_multi_asset", + "score": 0.843, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package", + "turn all of that into one reusable skill package" + ], + "eval_optimize": [ + "evals", + "packaging checks" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "runbook", + "process", + "prompt history" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.", + "family": "complex_multi_asset", + "score": 0.422, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.4, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "production-ready" + ], + "transform_workflow": [ + "workflow", + "workflow fragments" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.", + "family": "iterate_existing_skill", + "score": 0.557, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "distribution" + ], + "team_operationalize": [ + "team", + "distribution" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.", + "family": "paraphrase_trigger", + "score": 0.474, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.4, + "support_score": 0, + "lexical_support": 0.571, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "reuse_package": [ + "team library", + "maintained" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.", + "family": "paraphrase_trigger", + "score": 0.868, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0, + "lexical_support": 0.267, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this workflow" + ], + "eval_optimize": [ + "validate", + "route boundary" + ], + "reuse_package": [ + "package", + "distribution", + "shared capability" + ], + "team_operationalize": [ + "shared", + "distribution" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.", + "family": "long_context_trigger", + "score": 0.889, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.855, + "support_score": 0.06, + "lexical_support": 0.346, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill", + "skill package" + ], + "eval_optimize": [ + "validation" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reuse", + "package", + "team reuse" + ], + "team_operationalize": [ + "team", + "team reuse" + ], + "transform_workflow": [ + "rough notes", + "transcript", + "onboarding flow" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Give me ideas for improving our process.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.24, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "process" + ] + }, + "negative": { + "brainstorm_only": [ + "ideas" + ] + } + } + } + }, + { + "prompt": "Review this note and tell me what it means in plain English.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "tell me what it means", + "plain english" + ] + } + } + } + }, + { + "prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.", + "family": "brainstorm_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0.06, + "lexical_support": 0.2, + "negative_penalty": 0.66, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm", + "improve packaging" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Translate these notes into French and keep the structure exactly as-is.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "translate_only" + ], + "exclusive_negative_concepts": [ + "translate_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "translate_only": [ + "translate", + "into french" + ] + } + } + } + }, + { + "prompt": "Summarize this workflow and list the main points only.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.111, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "summarize_only": [ + "summarize" + ] + } + } + } + }, + { + "prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.174, + "negative_penalty": 0.68, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "explain_only": [ + "explain it", + "in simpler language" + ], + "no_build_directive": [ + "do not convert it into a reusable capability", + "keep it as documentation only" + ] + } + } + } + }, + { + "prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.", + "family": "long_context_summary_only", + "score": 0.256, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "summarize_only" + ], + "exclusive_negative_concepts": [ + "summarize_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.333, + "negative_penalty": 0.3, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "summarize_only": [ + "summary", + "recap" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Review this process note and explain it, no packaging needed.", + "family": "explain_not_package", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "explain_only": [ + "explain it" + ], + "no_build_directive": [ + "no packaging needed" + ] + } + } + } + }, + { + "prompt": "Turn this into a checklist and template, but stop short of making a full skill.", + "family": "partial_scaffold_not_full_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0.42, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "no_build_directive": [ + "stop short of making a full skill" + ] + } + } + } + }, + { + "prompt": "Package this explanation as a document, not as an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document" + ], + "no_build_directive": [ + "not as an agent skill" + ] + } + } + } + }, + { + "prompt": "Make a future-ready outline for this skill idea, but do not build the package.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "iterate_existing_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.327, + "support_score": 0, + "lexical_support": 0.267, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "iterate_existing_skill": [ + "this skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "reusable" + ] + }, + "negative": { + "document_only": [ + "document", + "reusable-looking document" + ], + "no_build_directive": [ + "without turning them into an agent skill" + ] + } + } + } + }, + { + "prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.238, + "negative_penalty": 0.82, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document", + "template only" + ], + "future_outline": [ + "not now", + "later but not now" + ], + "no_build_directive": [ + "not now", + "later but not now" + ] + } + } + } + }, + { + "prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.263, + "negative_penalty": 0.64, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "future_outline": [ + "for later", + "future-ready outline" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + } + ] + } + }, + "current_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 5, + "passed": 5, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + } + }, + "family_stats": { + "complex_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "iterate_existing_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "workflow_to_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "paraphrase_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_trigger": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "summary_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "translate_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "future_outline_vs_build": { + "total": 2, + "passed": 2, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "document_export_vs_agent_skill": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "long_context_near_neighbor": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "family": "complex_multi_asset", + "score": 0.468, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.4, + "support_score": 0.06, + "lexical_support": 0.316, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable" + ], + "team_operationalize": [ + "internal" + ], + "transform_workflow": [ + "process", + "internal process note" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.", + "family": "iterate_existing_skill", + "score": 0.727, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "iterate_existing_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.691, + "support_score": 0, + "lexical_support": 0.389, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "evals", + "near-neighbor", + "trigger boundary" + ], + "iterate_existing_skill": [ + "existing skill" + ], + "reuse_package": [ + "package", + "distribution" + ], + "team_operationalize": [ + "distribution" + ], + "transform_workflow": [ + "existing skill draft" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.", + "family": "workflow_to_skill", + "score": 0.568, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.509, + "support_score": 0, + "lexical_support": 0.5, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "eval_optimize": [ + "validation" + ], + "reuse_package": [ + "reusable", + "library entry" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.", + "family": "paraphrase_trigger", + "score": 0.858, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0.06, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "packaging checks" + ], + "multi_asset": [ + "references", + "scripts" + ], + "reuse_package": [ + "reusable", + "package", + "packaging" + ], + "transform_workflow": [ + "quarterly release routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.", + "family": "long_context_trigger", + "score": 0.744, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "team_operationalize", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.709, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "turn this playbook into an agent capability package" + ], + "reuse_package": [ + "package", + "team library" + ], + "team_operationalize": [ + "team" + ], + "transform_workflow": [ + "prompt history", + "transcript", + "release notes" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.", + "family": "summary_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.286, + "negative_penalty": 0.72, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "no_build_directive": [ + "not a reusable skill" + ], + "summarize_only": [ + "summary", + "only want a summary" + ] + } + } + } + }, + { + "prompt": "Explain this release checklist in simpler language; do not package it.", + "family": "explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "explain_only", + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.091, + "negative_penalty": 0.68, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "checklist" + ] + }, + "negative": { + "explain_only": [ + "explain this", + "in simpler language" + ], + "no_build_directive": [ + "do not package it" + ] + } + } + } + }, + { + "prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.", + "family": "long_context_document_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.222, + "negative_penalty": 0.6, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "document", + "wiki", + "reference document" + ], + "no_build_directive": [ + "not for agent execution" + ] + } + } + } + }, + { + "prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.", + "family": "translate_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "translate_only" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.77, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "no_build_directive": [ + "no skill package", + "keep it as documentation only" + ], + "translate_only": [ + "translate", + "into russian" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Help me shape an idea before we decide whether to build a skill.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline" + ], + "semantic_coverage": 0.309, + "support_score": 0, + "lexical_support": 0.308, + "negative_penalty": 0.46, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "build_skill": [ + "build a skill" + ] + }, + "negative": { + "brainstorm_only": [ + "shape an idea" + ], + "future_outline": [ + "before we decide whether to build" + ] + } + } + } + }, + { + "prompt": "Create an outline for a possible future skill, but do not build the skill yet.", + "family": "future_outline_vs_build", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill", + "outline for a possible future skill" + ], + "no_build_directive": [ + "do not build" + ] + } + } + } + }, + { + "prompt": "Turn this into a document package for the wiki, not an agent skill.", + "family": "document_export_vs_agent_skill", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.491, + "support_score": 0, + "lexical_support": 0.615, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "build_skill": [ + "agent skill" + ], + "reuse_package": [ + "package" + ] + }, + "negative": { + "document_only": [ + "document", + "document package", + "wiki" + ], + "no_build_directive": [ + "not an agent skill" + ] + } + } + } + }, + { + "prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.", + "family": "long_context_near_neighbor", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.673, + "support_score": 0, + "lexical_support": 0.438, + "negative_penalty": 0.6, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "workflow", + "checklist" + ] + }, + "negative": { + "document_only": [ + "template only" + ], + "no_build_directive": [ + "no skill package", + "discussion only" + ] + } + } + } + } + ] + } + }, + "winner_blind_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "score": 0.392, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.214, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "score": 0.405, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + }, + "boundary_case": true + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "score": 0.028, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.182, + "support_score": 0.06, + "lexical_support": 0.318, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + } + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.3, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "future_outline" + ], + "exclusive_negative_concepts": [ + "future_outline" + ], + "semantic_coverage": 0.0, + "support_score": 0.06, + "lexical_support": 0.308, + "negative_penalty": 0.22, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + } + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "score": 0.026, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline" + ], + "exclusive_negative_concepts": [ + "future_outline" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.273, + "negative_penalty": 0.22, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + } + } + ] + } + }, + "current_blind_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "score": 0.392, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.214, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "score": 0.405, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.375, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + }, + "boundary_case": true + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "score": 0.028, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.182, + "support_score": 0.06, + "lexical_support": 0.318, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + } + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.3, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "future_outline" + ], + "exclusive_negative_concepts": [ + "future_outline" + ], + "semantic_coverage": 0.0, + "support_score": 0.06, + "lexical_support": 0.308, + "negative_penalty": 0.22, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + } + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "score": 0.026, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline" + ], + "exclusive_negative_concepts": [ + "future_outline" + ], + "semantic_coverage": 0.364, + "support_score": 0, + "lexical_support": 0.273, + "negative_penalty": 0.22, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + } + } + ] + } + }, + "baseline_blind_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "score": 0.393, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.377, + "support_score": 0, + "lexical_support": 0.071, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + }, + "boundary_case": true + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "score": 0.392, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.377, + "support_score": 0, + "lexical_support": 0.062, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + }, + "boundary_case": true + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "score": 0.019, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only" + ], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.189, + "support_score": 0.06, + "lexical_support": 0.136, + "negative_penalty": 0.18, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + } + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "future_outline" + ], + "exclusive_negative_concepts": [ + "future_outline" + ], + "semantic_coverage": 0.0, + "support_score": 0.06, + "lexical_support": 0.0, + "negative_penalty": 0.22, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + } + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "score": 0.024, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline" + ], + "exclusive_negative_concepts": [ + "future_outline" + ], + "semantic_coverage": 0.377, + "support_score": 0, + "lexical_support": 0.091, + "negative_penalty": 0.22, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + } + } + ] + } + }, + "winner_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 0, + "margin": 3, + "confidence": 0.74, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow" + ], + "strong_positive_concepts": [ + "transform_workflow" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 3, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [ + "document_only" + ], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 1, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "prompts" + ], + "artifact_hits": [ + "prompts" + ], + "capability_overlap": 2, + "exclusive_negative_concepts": [ + "explain_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain", + "plain english" + ], + "positive_vote": 2, + "negative_vote": 3, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 2, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 2, + "margin": 1, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "current_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 0, + "margin": 3, + "confidence": 0.74, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow" + ], + "strong_positive_concepts": [ + "transform_workflow" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 3, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [ + "document_only" + ], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 1, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "prompts" + ], + "artifact_hits": [ + "prompts" + ], + "capability_overlap": 2, + "exclusive_negative_concepts": [ + "explain_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain", + "plain english" + ], + "positive_vote": 2, + "negative_vote": 3, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 2, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 2, + "margin": 1, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "baseline_judge_blind_holdout_report": { + "judge": "rubric-blind-v1", + "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "blind_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_multi_asset": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_document_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_explain_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_future_outline": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "blind_brainstorm_only": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.", + "family": "blind_operationalize", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 0, + "margin": 3, + "confidence": 0.74, + "concept_evidence": { + "positive": { + "reuse_package": [ + "maintained", + "operationalize" + ], + "transform_workflow": [ + "support escalation routine" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.", + "family": "blind_multi_asset", + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 2, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 4, + "negative_vote": 0, + "margin": 4, + "confidence": 0.82, + "concept_evidence": { + "positive": { + "reuse_package": [ + "reusable", + "package" + ], + "transform_workflow": [ + "sop", + "transcript" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.", + "family": "blind_document_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow" + ], + "strong_positive_concepts": [ + "transform_workflow" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 3, + "exclusive_negative_concepts": [], + "nonexclusive_negative_concepts": [ + "document_only" + ], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 2, + "negative_vote": 1, + "margin": 1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": { + "document_only": [ + "wiki" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Explain the differences between prompts and skills in plain English.", + "family": "blind_explain_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "prompts" + ], + "artifact_hits": [ + "prompts" + ], + "capability_overlap": 2, + "exclusive_negative_concepts": [ + "explain_only" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [ + "explain", + "plain english" + ], + "positive_vote": 2, + "negative_vote": 3, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": {}, + "negative": { + "explain_only": [ + "plain english" + ] + } + } + }, + "boundary_case": true + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future skill outline from these notes, but stop before generating files.", + "family": "blind_future_outline", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [], + "strong_positive_concepts": [], + "trigger_action_hits": [], + "input_hits": [ + "notes" + ], + "artifact_hits": [], + "capability_overlap": 0, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 1, + "negative_vote": 2, + "margin": -1, + "confidence": 0.58, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ] + }, + "negative": { + "future_outline": [ + "future skill" + ] + } + } + }, + "boundary_case": true + }, + { + "prompt": "Review this process and suggest improvements for later; no packaging yet.", + "family": "blind_brainstorm_only", + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "judge_detail": { + "mode": "judge-rubric", + "focused_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "strong_positive_concepts": [ + "transform_workflow", + "reuse_package" + ], + "trigger_action_hits": [], + "input_hits": [], + "artifact_hits": [], + "capability_overlap": 1, + "exclusive_negative_concepts": [ + "future_outline" + ], + "nonexclusive_negative_concepts": [], + "exclusion_hits": [], + "generic_exclusion_hits": [], + "positive_vote": 3, + "negative_vote": 2, + "margin": 1, + "confidence": 0.66, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ], + "transform_workflow": [ + "process" + ] + }, + "negative": { + "future_outline": [ + "for later" + ] + } + } + }, + "boundary_case": true + } + ] + }, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "winner_adversarial_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_multi_asset_boundary": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_document_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_summary_translation_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_outline_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_brainstorm_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", + "family": "adversarial_noisy_operationalize", + "score": 0.834, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.263, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "hardening" + ], + "reuse_package": [ + "package", + "maintained", + "operationalize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", + "family": "adversarial_multi_asset_boundary", + "score": 0.842, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.368, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this routine" + ], + "eval_optimize": [ + "stress test", + "route boundary" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "prompt history" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Create a wiki package from this runbook and do not generate any skill files.", + "family": "adversarial_document_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.429, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "document_only": [ + "wiki" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", + "family": "adversarial_summary_translation_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive", + "summarize_only", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only", + "translate_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.188, + "negative_penalty": 1.25, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "document_only": [ + "knowledge base" + ], + "no_build_directive": [ + "keep it as documentation only" + ], + "summarize_only": [ + "summarize" + ], + "translate_only": [ + "translate", + "into japanese" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", + "family": "adversarial_future_outline_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill" + ], + "no_build_directive": [ + "stop short of making a full skill", + "plain note", + "keep it as a plain note" + ] + } + } + } + }, + { + "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", + "family": "adversarial_brainstorm_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.182, + "negative_penalty": 0.88, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ], + "future_outline": [ + "for later" + ], + "no_build_directive": [ + "discussion only" + ] + } + } + } + } + ] + } + }, + "current_adversarial_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_multi_asset_boundary": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_document_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_summary_translation_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_outline_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_brainstorm_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", + "family": "adversarial_noisy_operationalize", + "score": 0.834, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.263, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "hardening" + ], + "reuse_package": [ + "package", + "maintained", + "operationalize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", + "family": "adversarial_multi_asset_boundary", + "score": 0.842, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.368, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this routine" + ], + "eval_optimize": [ + "stress test", + "route boundary" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "prompt history" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Create a wiki package from this runbook and do not generate any skill files.", + "family": "adversarial_document_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.429, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "document_only": [ + "wiki" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", + "family": "adversarial_summary_translation_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive", + "summarize_only", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only", + "translate_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.188, + "negative_penalty": 1.25, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "document_only": [ + "knowledge base" + ], + "no_build_directive": [ + "keep it as documentation only" + ], + "summarize_only": [ + "summarize" + ], + "translate_only": [ + "translate", + "into japanese" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", + "family": "adversarial_future_outline_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill" + ], + "no_build_directive": [ + "stop short of making a full skill", + "plain note", + "keep it as a plain note" + ] + } + } + } + }, + { + "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", + "family": "adversarial_brainstorm_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.182, + "negative_penalty": 0.88, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ], + "future_outline": [ + "for later" + ], + "no_build_directive": [ + "discussion only" + ] + } + } + } + } + ] + } + }, + "baseline_adversarial_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_multi_asset_boundary": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_document_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_summary_translation_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_outline_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_brainstorm_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", + "family": "adversarial_noisy_operationalize", + "score": 0.845, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0, + "lexical_support": 0.053, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "hardening" + ], + "reuse_package": [ + "package", + "maintained", + "operationalize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", + "family": "adversarial_multi_asset_boundary", + "score": 0.845, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0, + "lexical_support": 0.053, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this routine" + ], + "eval_optimize": [ + "stress test", + "route boundary" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "prompt history" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Create a wiki package from this runbook and do not generate any skill files.", + "family": "adversarial_document_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.377, + "support_score": 0.06, + "lexical_support": 0.143, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "document_only": [ + "wiki" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", + "family": "adversarial_summary_translation_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive", + "summarize_only", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only", + "translate_only" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.062, + "negative_penalty": 1.25, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "document_only": [ + "knowledge base" + ], + "no_build_directive": [ + "keep it as documentation only" + ], + "summarize_only": [ + "summarize" + ], + "translate_only": [ + "translate", + "into japanese" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", + "family": "adversarial_future_outline_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.05, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill" + ], + "no_build_directive": [ + "stop short of making a full skill", + "plain note", + "keep it as a plain note" + ] + } + } + } + }, + { + "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", + "family": "adversarial_brainstorm_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.091, + "negative_penalty": 0.88, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ], + "future_outline": [ + "for later" + ], + "no_build_directive": [ + "discussion only" + ] + } + } + } + } + ] + } + }, + "candidates": [ + { + "id": "current", + "label": "Current", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "strategy": "current", + "estimated_tokens": 65, + "dev": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "selection_key": [ + 0, + 0, + -1.0, + -1.0, + -1.0, + -1.0, + 65 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + { + "id": "minimal", + "label": "Minimal", + "description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "strategy": "minimal_template", + "estimated_tokens": 41, + "dev": { + "false_positives": 1, + "false_negatives": 1, + "precision": 0.875, + "recall": 0.875, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 0.857 + }, + "selection_key": [ + 1, + 1, + -1.0, + -0.857, + -0.875, + -0.875, + 41 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + { + "id": "guardrail", + "label": "Guardrail", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", + "strategy": "guardrail_template", + "estimated_tokens": 56, + "dev": { + "false_positives": 1, + "false_negatives": 1, + "precision": 0.875, + "recall": 0.875, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 0.857 + }, + "selection_key": [ + 1, + 1, + -1.0, + -0.857, + -0.875, + -0.875, + 56 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + { + "id": "balanced", + "label": "Balanced", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "strategy": "balanced_template", + "estimated_tokens": 60, + "dev": { + "false_positives": 1, + "false_negatives": 1, + "precision": 0.875, + "recall": 0.875, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 0.857 + }, + "selection_key": [ + 1, + 1, + -1.0, + -0.857, + -0.875, + -0.875, + 60 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + { + "id": "artifact_aware", + "label": "Artifact Aware", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "strategy": "artifact_template", + "estimated_tokens": 77, + "dev": { + "false_positives": 1, + "false_negatives": 1, + "precision": 0.875, + "recall": 0.875, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 0.857 + }, + "selection_key": [ + 1, + 1, + -1.0, + -0.857, + -0.875, + -0.875, + 77 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + }, + { + "id": "boundary", + "label": "Boundary", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", + "strategy": "boundary_template", + "estimated_tokens": 83, + "dev": { + "false_positives": 1, + "false_negatives": 1, + "precision": 0.875, + "recall": 0.875, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 0.857 + }, + "selection_key": [ + 1, + 1, + -1.0, + -0.857, + -0.875, + -0.875, + 83 + ], + "holdout": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + } + } + ], + "selection_logic": { + "priority": [ + "fewest false positives", + "fewest false negatives", + "highest near-neighbor pass rate", + "highest negative pass rate", + "highest precision", + "highest recall", + "shortest description" + ] + }, + "comparison": { + "winner_vs_current_dev": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_current_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_current_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_dev": { + "baseline_false_positives": 0, + "baseline_false_negatives": 1, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": -1, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 0.875, + "improved_recall": 1.0 + }, + "winner_vs_baseline_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_current_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_judge_blind_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_current_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + } + }, + "acceptance_gates": { + "selection_basis": "dev only", + "holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.673, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "current_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.673, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "baseline_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.648, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.416, + "max_non_trigger_score": 0.0, + "score_gap": 0.416, + "threshold_margin": 0.086, + "positive_threshold_buffer": 0.086, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "winner_family_health": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "blind_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.028, + "score_gap": 0.364, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.302, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "current_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.028, + "score_gap": 0.364, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.302, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "baseline_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.393, + "mean_non_trigger_score": 0.011, + "mean_near_neighbor_score": 0.012, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.024, + "score_gap": 0.368, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.306, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "judge_blind_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0, + "judge_summary": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "winner_calibration": null, + "current_calibration": null, + "baseline_calibration": null, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "adversarial_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "current_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "baseline_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.845, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.845, + "max_non_trigger_score": 0.0, + "score_gap": 0.845, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.515, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + } + }, + "summary": { + "winner_label": "Current", + "winner_tokens": 65, + "current_tokens": 65, + "winner_dev_total_errors": 0, + "current_dev_total_errors": 0, + "winner_holdout_total_errors": 0, + "current_holdout_total_errors": 0, + "winner_blind_holdout_total_errors": 0, + "current_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_total_errors": 0, + "current_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_total_errors": 0, + "current_adversarial_holdout_total_errors": 0, + "winner_judge_blind_agreement_rate": 1.0, + "winner_adversarial_risk_band": "healthy", + "winner_adversarial_score_gap": 0.834, + "candidate_count": 6, + "baseline_tokens": 8, + "baseline_dev_total_errors": 1, + "baseline_holdout_total_errors": 0, + "baseline_blind_holdout_total_errors": 0, + "baseline_judge_blind_holdout_total_errors": 0, + "baseline_adversarial_holdout_total_errors": 0 + } + }, + "promotion": { + "ok": true, + "summary": { + "promote": 0, + "keep_current": 3, + "blocked": 0 + }, + "targets": [ + { + "name": "yao-meta-skill", + "winner_label": "Current", + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "winner_tokens": 65, + "current_tokens": 65, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.33, + "mean_positive_score": 0.673, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "blind_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.028, + "score_gap": 0.364, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.302, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "adversarial_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.66, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", + "ok": true, + "promotion": { + "decision": "keep_current", + "winner_changed": false, + "all_gates_pass": true, + "route_clean": true, + "judge_agreement_ok": true, + "causes": [ + "no_candidate_outperformed_current" + ], + "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", + "gate_status": { + "visible_holdout_non_regression": true, + "blind_holdout_non_regression": true, + "judge_blind_non_regression": true, + "judge_blind_agreement": true, + "adversarial_non_regression": true, + "adversarial_risk_ok": true, + "route_confusion_clean": true, + "family_stability": true + }, + "route_metrics": { + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.611, + "misroute_count": 0, + "ambiguous_case_count": 0 + } + }, + "registry": [ + { + "name": "yao-meta-skill", + "role": "baseline", + "label": "Baseline", + "ranking_state": "reference", + "promotion_state": "reference", + "description": "Create and improve agent skills.", + "tokens": 8, + "dev_errors": 1, + "holdout_errors": 0 + }, + { + "name": "yao-meta-skill", + "role": "current", + "label": "Current", + "ranking_state": "selected_by_dev", + "promotion_state": "kept_current", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "tokens": 65, + "dev_errors": 0, + "holdout_errors": 0, + "reason_tags": [] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Minimal", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "tokens": 41, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", + "tokens": 56, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Balanced", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "tokens": 60, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Artifact Aware", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.", + "tokens": 77, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit", + "longer_without_gain" + ] + }, + { + "name": "yao-meta-skill", + "role": "candidate", + "label": "Boundary", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.", + "tokens": 83, + "dev_errors": 2, + "holdout_errors": 0, + "reason_tags": [ + "weaker_dev_fit", + "longer_without_gain" + ] + } + ], + "decision": "keep_current" + }, + { + "name": "team-frontend-review", + "winner_label": "Current", + "winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "winner_tokens": 50, + "current_tokens": 50, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.685, + "mean_non_trigger_score": 0.089, + "mean_near_neighbor_score": 0.045, + "min_positive_score": 0.496, + "max_non_trigger_score": 0.211, + "score_gap": 0.285, + "threshold_margin": 0.016, + "positive_threshold_buffer": 0.016, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.685, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.226, + "min_positive_score": 0.504, + "max_non_trigger_score": 0.416, + "score_gap": 0.088, + "threshold_margin": 0.024, + "positive_threshold_buffer": 0.024, + "negative_threshold_buffer": 0.064, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.71, + "mean_non_trigger_score": 0.009, + "mean_near_neighbor_score": 0.018, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.035, + "score_gap": 0.509, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.305, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.703, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", + "ok": true, + "promotion": { + "decision": "keep_current", + "winner_changed": false, + "all_gates_pass": true, + "route_clean": true, + "judge_agreement_ok": true, + "causes": [ + "no_candidate_outperformed_current", + "current_holdout_risk" + ], + "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", + "gate_status": { + "visible_holdout_non_regression": true, + "blind_holdout_non_regression": true, + "judge_blind_non_regression": true, + "judge_blind_agreement": true, + "adversarial_non_regression": true, + "adversarial_risk_ok": true, + "route_confusion_clean": true, + "family_stability": true + }, + "route_metrics": { + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.805, + "misroute_count": 0, + "ambiguous_case_count": 0 + } + }, + "registry": [ + { + "name": "team-frontend-review", + "role": "baseline", + "label": "Baseline", + "ranking_state": "reference", + "promotion_state": "reference", + "description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.", + "tokens": 52, + "dev_errors": 3, + "holdout_errors": 0 + }, + { + "name": "team-frontend-review", + "role": "current", + "label": "Current", + "ranking_state": "selected_by_dev", + "promotion_state": "kept_current", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "tokens": 50, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.", + "tokens": 62, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Balanced", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "tokens": 64, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Artifact Aware", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Trigger when requests mention React components, forms, frontend diffs, or UI code and the job is to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "tokens": 84, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "team-frontend-review", + "role": "candidate", + "label": "Boundary", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.", + "tokens": 90, + "dev_errors": 3, + "holdout_errors": 0, + "reason_tags": [ + "longer_without_gain" + ] + } + ], + "decision": "keep_current" + }, + { + "name": "governed-incident-command", + "winner_label": "Current", + "winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "winner_tokens": 37, + "current_tokens": 37, + "winner_holdout_fp": 0, + "winner_holdout_fn": 1, + "current_holdout_fp": 0, + "current_holdout_fn": 1, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 2, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 1, + "winner_blind_holdout_total_errors": 0, + "winner_judge_blind_holdout_fp": 0, + "winner_judge_blind_holdout_fn": 0, + "current_judge_blind_holdout_fp": 0, + "current_judge_blind_holdout_fn": 0, + "baseline_judge_blind_holdout_fp": 0, + "baseline_judge_blind_holdout_fn": 0, + "winner_judge_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.37, + "mean_non_trigger_score": 0.019, + "mean_near_neighbor_score": 0.037, + "min_positive_score": 0.08, + "max_non_trigger_score": 0.057, + "score_gap": 0.023, + "threshold_margin": -0.4, + "positive_threshold_buffer": -0.4, + "negative_threshold_buffer": 0.423, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "overlap" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.902, + "mean_non_trigger_score": 0.168, + "mean_near_neighbor_score": 0.138, + "min_positive_score": 0.804, + "max_non_trigger_score": 0.386, + "score_gap": 0.418, + "threshold_margin": 0.094, + "positive_threshold_buffer": 0.324, + "negative_threshold_buffer": 0.094, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "healthy" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.827, + "mean_non_trigger_score": 0.027, + "mean_near_neighbor_score": 0.011, + "min_positive_score": 0.654, + "max_non_trigger_score": 0.056, + "score_gap": 0.598, + "threshold_margin": 0.284, + "positive_threshold_buffer": 0.314, + "negative_threshold_buffer": 0.284, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "judge_blind": { + "winner": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "current": { + "agreement_rate": 1.0, + "mean_confidence": 0.657, + "rubric_version": "blind-v1" + }, + "baseline": { + "agreement_rate": 1.0, + "mean_confidence": 0.69, + "rubric_version": "blind-v1" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "judge_blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, judge-backed blind, adversarial, and calibration gates active", + "ok": true, + "promotion": { + "decision": "keep_current", + "winner_changed": false, + "all_gates_pass": true, + "route_clean": true, + "judge_agreement_ok": true, + "causes": [ + "no_candidate_outperformed_current", + "current_holdout_gap_present", + "current_holdout_risk" + ], + "next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.", + "gate_status": { + "visible_holdout_non_regression": true, + "blind_holdout_non_regression": true, + "judge_blind_non_regression": true, + "judge_blind_agreement": true, + "adversarial_non_regression": true, + "adversarial_risk_ok": true, + "route_confusion_clean": true, + "family_stability": true + }, + "route_metrics": { + "precision": 1.0, + "recall": 1.0, + "average_margin": 0.659, + "misroute_count": 0, + "ambiguous_case_count": 0 + } + }, + "registry": [ + { + "name": "governed-incident-command", + "role": "baseline", + "label": "Baseline", + "ranking_state": "reference", + "promotion_state": "reference", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.", + "tokens": 93, + "dev_errors": 1, + "holdout_errors": 2 + }, + { + "name": "governed-incident-command", + "role": "current", + "label": "Current", + "ranking_state": "selected_by_dev", + "promotion_state": "kept_current", + "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "tokens": 37, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Guardrail", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 51, + "dev_errors": 1, + "holdout_errors": 2, + "reason_tags": [ + "visible_holdout_regression", + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Balanced", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "tokens": 54, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Boundary", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.", + "tokens": 78, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + }, + { + "name": "governed-incident-command", + "role": "candidate", + "label": "Artifact Aware", + "ranking_state": "not_selected", + "promotion_state": "blocked", + "description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.", + "tokens": 78, + "dev_errors": 1, + "holdout_errors": 1, + "reason_tags": [ + "longer_without_gain" + ] + } + ], + "decision": "keep_current" + } + ] + }, + "benchmark": { + "ok": true, + "query": "skills", + "source": "github-api", + "network_boundary": "github-api-only", + "repositories": [ + { + "full_name": "obra/superpowers", + "html_url": "https://github.com/obra/superpowers", + "description": "An agentic skills framework & software development methodology that works.", + "stars": 226125, + "topics": [], + "primary_language": "Shell", + "patterns": [ + { + "category": "method", + "matched": "process", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale." + }, + { + "category": "execution", + "matched": "cli", + "borrow": "Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not copy repo-specific commands or environment assumptions verbatim." + }, + { + "category": "evaluation", + "matched": "test", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here." + }, + { + "category": "structure", + "matched": "docs", + "borrow": "Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not mirror documentation bulk that adds context cost without improving reliability." + } + ], + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure.", + "Borrow explicit validation and quality gates that make iteration safer." + ], + "avoid": [ + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim.", + "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here." + ], + "readme_excerpt": "# Superpowers\n\nSuperpowers is a complete software development methodology for your coding agents, built on top of a set of composable skills and some initial instructions that make sure your agent uses them.\n\n## Quickstart\n\nGive your agent Superpowers: [Claude Code](#claude-code), [Codex CLI](#codex-cli), [Codex App](#codex-app), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [OpenCode](#opencode), [Cursor](#cursor), [GitHub Copilot CLI](#github-copilot-cli).\n\n## How it works\n\nIt starts from the moment you fire up your coding agent. As soon as it sees that you're building something, it *doesn't* just jump into trying to write code. Instead, it steps back and asks you what you're really trying to do. \n\nOnce it's teased a spec out of the conversation, it shows it to you in chunks short enough to actually read and digest. \n\nAfter you've signed off on the design, your agent puts together an implementation plan that's clear enough for an enthusiastic junior engineer with poor taste, no judgement, no project context, and an aversion to testing to follow. It emphasizes true red/green TDD, YAGNI (You Aren't Gonna Need It), and DRY. \n\nNext up, once you say \"go\", it launches a *" + }, + { + "full_name": "affaan-m/ECC", + "html_url": "https://github.com/affaan-m/ECC", + "description": "The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.", + "stars": 214381, + "topics": [ + "ai-agents", + "anthropic", + "claude", + "claude-code", + "developer-tools", + "llm", + "mcp", + "productivity" + ], + "primary_language": "JavaScript", + "patterns": [ + { + "category": "method", + "matched": "workflow", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale." + }, + { + "category": "execution", + "matched": "command, shell, script", + "borrow": "Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not copy repo-specific commands or environment assumptions verbatim." + }, + { + "category": "structure", + "matched": "guide, docs", + "borrow": "Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not mirror documentation bulk that adds context cost without improving reliability." + }, + { + "category": "governance", + "matched": "security, review", + "borrow": "Borrow the explicit review, safety, or operational trust boundaries.", + "avoid": "Do not import compliance-heavy process where the new skill does not need it." + } + ], + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure.", + "Borrow the way it separates explanation, examples, and reusable structure." + ], + "avoid": [ + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim.", + "Do not mirror documentation bulk that adds context cost without improving reliability." + ], + "readme_excerpt": "**Language:** English | [Português (Brasil)](docs/pt-BR/README.md) | [简体中文](README.zh-CN.md) | [繁體中文](docs/zh-TW/README.md) | [日本語](docs/ja-JP/README.md) | [한국어](docs/ko-KR/README.md) | [Türkçe](docs/tr/README.md) | [Русский](docs/ru/README.md) | [Tiếng Việt](docs/vi-VN/README.md) | [ไทย](docs/th/README.md) | [Deutsch](docs/de-DE/README.md) | [Español](docs/es/README.md)\n\n![ECC — the agent harness operating system](assets/hero.png)\n\n[![Stars](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fstars&style=flat)](https://github.com/affaan-m/ECC/stargazers)\n[![Forks](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fforks&style=flat)](https://github.com/affaan-m/ECC/network/members)\n[![Contributors](https://img.shields.io/github/contributors/affaan-m/ECC?style=flat)](https://github.com/affaan-m/ECC/graphs/contributors)\n[![npm ecc-universal](https://img.shields.io/npm/dw/ecc-universal?label=ecc-universal%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-universal)\n[![npm ecc-agentshield](https://img.shields.io/npm/dw/ecc-agentshield?label=ecc-agentshield%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-agen" + }, + { + "full_name": "multica-ai/andrej-karpathy-skills", + "html_url": "https://github.com/multica-ai/andrej-karpathy-skills", + "description": "A single CLAUDE.md file to improve Claude Code behavior, derived from Andrej Karpathy's observations on LLM coding pitfalls.", + "stars": 174264, + "topics": [], + "primary_language": null, + "patterns": [ + { + "category": "evaluation", + "matched": "test, validation", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here." + }, + { + "category": "structure", + "matched": "guide", + "borrow": "Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not mirror documentation bulk that adds context cost without improving reliability." + }, + { + "category": "portability", + "matched": "plugin", + "borrow": "Borrow how it keeps core semantics stable across environments or integrations.", + "avoid": "Do not inherit platform lock-in or vendor-specific branching unless truly required." + } + ], + "borrow": [ + "Borrow explicit validation and quality gates that make iteration safer.", + "Borrow the way it separates explanation, examples, and reusable structure.", + "Borrow how it keeps core semantics stable across environments or integrations." + ], + "avoid": [ + "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "Do not mirror documentation bulk that adds context cost without improving reliability.", + "Do not inherit platform lock-in or vendor-specific branching unless truly required." + ], + "readme_excerpt": "# Karpathy-Inspired Claude Code Guidelines\n\n> Check out my new project [Multica](https://github.com/multica-ai/multica) — an open-source platform for running and managing coding agents with reusable skills.\n>\n> Follow me on X: [https://x.com/jiayuan_jy](https://x.com/jiayuan_jy)\n\nA single `CLAUDE.md` file to improve Claude Code behavior, derived from [Andrej Karpathy's observations](https://x.com/karpathy/status/2015883857489522876) on LLM coding pitfalls.\n\nEnglish | [简体中文](./README.zh.md)\n\n## The Problems\n\nFrom Andrej's post:\n\n> \"The models make wrong assumptions on your behalf and just run along with them without checking. They don't manage their confusion, don't seek clarifications, don't surface inconsistencies, don't present tradeoffs, don't push back when they should.\"\n\n> \"They really like to overcomplicate code and APIs, bloat abstractions, don't clean up dead code... implement a bloated construction over 1000 lines when 100 would do.\"\n\n> \"They still sometimes change/remove comments and code they don't sufficiently understand as side effects, even if orthogonal to the task.\"\n\n## The Solution\n\nFour principles in one file that directly address these issues:\n\n| Principle | Addr" + } + ], + "cross_repo": { + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure.", + "Borrow explicit validation and quality gates that make iteration safer.", + "Borrow the way it separates explanation, examples, and reusable structure.", + "Borrow how it keeps core semantics stable across environments or integrations.", + "Ask the user which of these patterns feels most worth borrowing before freezing the package shape." + ], + "avoid": [ + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim.", + "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "Do not mirror documentation bulk that adds context cost without improving reliability.", + "Do not inherit platform lock-in or vendor-specific branching unless truly required." + ] + }, + "external_references": [ + { + "name": "obra/superpowers", + "category": "method", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "source": "external" + }, + { + "name": "affaan-m/ECC", + "category": "method", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "source": "external" + }, + { + "name": "multica-ai/andrej-karpathy-skills", + "category": "evaluation", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "source": "external" + } + ], + "borrow_prompt": "I found 3 public GitHub projects worth studying. Do you want to borrow any of these patterns for method, structure, execution, or portability?", + "warnings": [] + }, + "reference_synthesis": { + "skill_name": "yao-meta-skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "intent_confidence": { + "score": 30, + "band": "low", + "gate_passed": false + }, + "github_benchmarks": [ + { + "name": "obra/superpowers", + "url": "https://github.com/obra/superpowers", + "stars": 226125, + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure." + ] + }, + { + "name": "affaan-m/ECC", + "url": "https://github.com/affaan-m/ECC", + "stars": 214381, + "borrow": [ + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure." + ] + }, + { + "name": "multica-ai/andrej-karpathy-skills", + "url": "https://github.com/multica-ai/andrej-karpathy-skills", + "stars": 174264, + "borrow": [ + "Borrow explicit validation and quality gates that make iteration safer.", + "Borrow the way it separates explanation, examples, and reusable structure." + ] + } + ], + "source_tracks": [ + { + "source_type": "official", + "name": "Official workflow product ergonomics", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "workflow" + ], + "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", + "why_relevant": "This track matches: workflow." + }, + { + "source_type": "research", + "name": "Hypothesis-test-learn loop", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + "why_relevant": "This track matches: general fit." + }, + { + "source_type": "principles", + "name": "Boundary-first design", + "evidence_mode": "curated-pattern-track", + "matched_keywords": [ + "general fit" + ], + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + "why_relevant": "This track matches: general fit." + } + ], + "synthesis": { + "borrow_now": [ + "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "Borrow the discipline of defining what the skill should not own before growing the package.", + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure." + ], + "avoid_now": [ + "Do not mimic product polish that adds UI bulk without improving clarity.", + "Do not create experimental overhead that exceeds the skill's real risk tier.", + "Do not expand execution assets until route boundaries stay clean.", + "Do not import process overhead that only exists for that project's scale.", + "Do not copy repo-specific commands or environment assumptions verbatim." + ], + "quality_risers": [ + "Use GitHub repositories for concrete package and workflow patterns.", + "Use curated official or commercial tracks for entrypoint and operator ergonomics.", + "Use research tracks to justify the smallest evaluation loop that still catches regressions.", + "Use principle tracks to keep the package small, boundary-aware, and outcome-driven." + ], + "pattern_gate": { + "threshold": 4, + "source_count": 6, + "accepted": [ + { + "name": "Official workflow product ergonomics", + "source_type": "official", + "borrow": "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "avoid": "Do not mimic product polish that adds UI bulk without improving clarity.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + }, + { + "name": "obra/superpowers", + "source_type": "github", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.; Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + }, + { + "name": "affaan-m/ECC", + "source_type": "github", + "borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.; Borrow the clear execution entrypoints and command structure.", + "avoid": "Do not import process overhead that only exists for that project's scale.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "distinctiveness", + "boundary" + ], + "missing": [], + "score": 4 + } + ], + "deferred": [ + { + "name": "Hypothesis-test-learn loop", + "source_type": "research", + "borrow": "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", + "avoid": "Do not create experimental overhead that exceeds the skill's real risk tier.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + }, + { + "name": "Boundary-first design", + "source_type": "principles", + "borrow": "Borrow the discipline of defining what the skill should not own before growing the package.", + "avoid": "Do not expand execution assets until route boundaries stay clean.", + "gates": { + "recurrence": true, + "generativity": true, + "distinctiveness": false, + "boundary": true + }, + "passed": [ + "recurrence", + "generativity", + "boundary" + ], + "missing": [ + "distinctiveness" + ], + "score": 3 + }, + { + "name": "multica-ai/andrej-karpathy-skills", + "source_type": "github", + "borrow": "Borrow explicit validation and quality gates that make iteration safer.; Borrow the way it separates explanation, examples, and reusable structure.", + "avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.", + "gates": { + "recurrence": true, + "generativity": false, + "distinctiveness": true, + "boundary": true + }, + "passed": [ + "recurrence", + "distinctiveness", + "boundary" + ], + "missing": [ + "generativity" + ], + "score": 3 + } + ], + "summary": "3 accepted, 3 deferred using threshold 4/4." + }, + "conflicts": [], + "recommendation": { + "summary": "Start by borrowing this pattern: Borrow a first-time operator flow that explains itself before it asks for more structure. Avoid this for the first pass: Do not mimic product polish that adds UI bulk without improving clarity.", + "borrow_now": [ + "Borrow a first-time operator flow that explains itself before it asks for more structure.", + "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed." + ], + "avoid_for_now": [ + "Do not mimic product polish that adds UI bulk without improving clarity.", + "Do not create experimental overhead that exceeds the skill's real risk tier." + ], + "why": "Intent still has gaps, so the system should surface the recommendation and ask for correction before deepening the package.", + "user_decision_required": true + }, + "visibility": { + "mode": "explicit", + "user_decision_required": true, + "reasons": [ + "intent_uncertain" + ], + "user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.", + "reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers." + }, + "decision_prompt": "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.", + "source_mix": { + "github_benchmarks": 3, + "curated_tracks": 3, + "user_references": 0 + } + } + }, + "output_risk": { + "skill_name": "yao-meta-skill", + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], + "top_risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap." + ], + "output_constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output." + ], + "self_repair_checks": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards.", + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note.", + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ], + "reviewer_note": "Use this report before deepening the package and again before approving example outputs." + }, + "artifact_design": { + "skill_name": "yao-meta-skill", + "design_system": "content-led editorial", + "primary_artifact": { + "key": "report", + "label": "Report or brief", + "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", + "matched_keywords": [ + "report", + "summary" + ] + }, + "artifact_families": [ + { + "key": "report", + "label": "Report or brief", + "score": 2, + "matched_keywords": [ + "report", + "summary" + ], + "direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." + }, + { + "key": "code_or_cli", + "label": "Code, CLI, or implementation guide", + "score": 1, + "matched_keywords": [ + "script" + ], + "direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit." + } + ], + "layout_patterns": [ + "thesis", + "evidence blocks", + "decision table", + "risks", + "next actions", + "prerequisites", + "commands", + "expected output" + ], + "design_tokens": { + "type": [ + "Use a distinctive display face or serif for major claims when the artifact is editorial.", + "Use a restrained sans for dense body text and technical details.", + "Use mono only for metadata, paths, commands, labels, and evidence tags." + ], + "color": [ + "Choose colors from the artifact's domain, brand, or evidence mood.", + "Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it.", + "Keep accent color limited to decisions, active states, risk, or section anchors." + ], + "spacing": [ + "Prefer clear grid rhythm over floating decorative cards.", + "Increase whitespace around decisions and shrink whitespace around supporting metadata.", + "Split dense content instead of shrinking type or adding scroll traps." + ], + "components": [ + "Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence.", + "Avoid cards inside cards.", + "Keep reviewer-only detail visible but visually quieter than user-facing guidance." + ] + }, + "quality_gates": [ + "Keep the first screen useful without requiring the reader to parse every detail.", + "Use tables only for comparisons; move explanations below the table.", + "Keep source notes readable without flooding the body with markers.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding." + ], + "anti_patterns": [ + "Do not copy Kami's fixed parchment background as a default.", + "Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them.", + "Do not let Markdown tables become the default shape for every comparison or explanation.", + "Do not turn reviewer evidence into user-facing clutter.", + "Do not invent screenshots, citations, charts, or UI states." + ], + "reviewer_note": "Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered." + }, + "prompt_quality": { + "skill_name": "yao-meta-skill", + "relevance": "prompt-heavy", + "primary_task_family": { + "key": "dialogue_interaction", + "label": "Dialogue interaction", + "matched_keywords": [ + "dialogue", + "conversation" + ] + }, + "task_families": [ + { + "key": "dialogue_interaction", + "label": "Dialogue interaction", + "score": 2, + "matched_keywords": [ + "dialogue", + "conversation" + ], + "role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", + "task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", + "format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions." + }, + { + "key": "execution_operation", + "label": "Execution operation", + "score": 1, + "matched_keywords": [ + "workflow" + ], + "role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.", + "task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.", + "format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant." + }, + { + "key": "teaching_guidance", + "label": "Teaching guidance", + "score": 1, + "matched_keywords": [ + "teach" + ], + "role_guidance": "Use a teacher role that adapts to learner level and avoids overloading the first pass.", + "task_guidance": "Explain through progressive steps, examples, and visible success checks.", + "format_guidance": "Return learner-facing sections, worked examples, checkpoints, and common mistakes." + }, + { + "key": "prompt_engineering", + "label": "Prompt engineering", + "score": 1, + "matched_keywords": [ + "prompt" + ], + "role_guidance": "Use a prompt engineer role only when role design materially improves execution.", + "task_guidance": "Map Role, Task, and Format into skill behavior rather than copying a large prompt template.", + "format_guidance": "Return a compact prompt contract plus tests, quality matrix, and usage notes." + } + ], + "complexity": { + "band": "complex", + "score": 6, + "reason": "multiple inputs, constraints, or task families require tradeoff handling" + }, + "need_model": { + "explicit_need": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "implicit_need": "The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.", + "scenario": "not yet explicit", + "user_level": "infer from examples and standards; ask only if it changes output depth", + "success_standard": "usable output with clear validation cues" + }, + "rtf_to_skill": { + "role": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.", + "task": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.", + "format": "Return concise prompts, decision points, and reviewer-visible assumptions." + }, + "quality_matrix": [ + { + "key": "completeness", + "label": "Completeness", + "score": 80, + "matched_signals": [ + "output", + "constraint", + "standard" + ], + "repair": "Name missing inputs, outputs, constraints, or success standards before deepening the package." + }, + { + "key": "clarity", + "label": "Clarity", + "score": 80, + "matched_signals": [], + "repair": "Replace broad verbs with observable actions and define what done means." + }, + { + "key": "consistency", + "label": "Consistency", + "score": 90, + "matched_signals": [ + "aligned", + "boundary" + ], + "repair": "Check that role, task, format, exclusions, and examples do not contradict each other." + }, + { + "key": "practicality", + "label": "Practicality", + "score": 90, + "matched_signals": [ + "use", + "workflow" + ], + "repair": "Add runnable steps, examples, or verification cues instead of abstract advice." + }, + { + "key": "specificity", + "label": "Specificity", + "score": 70, + "matched_signals": [], + "repair": "Anchor wording in the user's audience, domain nouns, and target outcome." + } + ], + "overall_quality_score": 82.0, + "self_repair_checks": [ + "Check explicit need, implicit need, scenario, user level, and success standard before deepening.", + "Map Role, Task, and Format into skill behavior, not decorative prompt labels.", + "Ask one focused clarification only when missing information changes the package boundary.", + "Add tests or examples for prompt-heavy behavior before treating it as reusable.", + "Keep prompt methodology in references and reports instead of bloating SKILL.md." + ], + "reviewer_note": "Use this profile when the package depends on prompt behavior, role design, output contracts, or conversation quality." + }, + "evidence_readiness": { + "score": 85, + "checks": [ + { + "label": "Intent clarity", + "status": "needs review", + "detail": "30/100 intent confidence." + }, + { + "label": "Benchmark coverage", + "status": "ready", + "detail": "3 GitHub benchmark repositories attached." + }, + { + "label": "Pattern gate", + "status": "ready", + "detail": "3 accepted, 3 deferred using threshold 4/4." + }, + { + "label": "Conflict handling", + "status": "ready", + "detail": "No material conflicts detected." + }, + { + "label": "Output risk profile", + "status": "ready", + "detail": "5 output risk families attached." + }, + { + "label": "Artifact design profile", + "status": "ready", + "detail": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail." + }, + { + "label": "Prompt quality profile", + "status": "ready", + "detail": "82.0/100 prompt-facing quality score." + } + ], + "reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery." + } +} \ No newline at end of file diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json new file mode 100644 index 00000000..8ecc4e58 --- /dev/null +++ b/reports/security_trust_report.json @@ -0,0 +1,433 @@ +{ + "ok": true, + "skill_dir": ".", + "summary": { + "scanned_files": 124, + "script_count": 49, + "secret_findings": 0, + "dependency_files": [ + "requirements-ci.txt" + ], + "network_script_count": 2, + "interactive_script_count": 0, + "package_sha256": "d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac" + }, + "failures": [], + "warnings": [ + "Scripts without argparse/help surface: scripts/render_context_reports.py, scripts/render_social_preview.py, scripts/skill_report_charts.py, scripts/skill_report_metrics.py, scripts/skill_report_model.py", + "Network-capable scripts require bounded host policy: scripts/check_update.py, scripts/github_benchmark_scan.py" + ], + "secrets": [], + "scripts": [ + { + "path": "scripts/build_confusion_matrix.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/build_skill_atlas.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/check_update.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": true, + "uses_subprocess": false + }, + { + "path": "scripts/ci_test.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/collect_feedback.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/context_sizer.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/create_iteration_snapshot.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/cross_packager.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/diff_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/export_skill_ir.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/github_benchmark_scan.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": true, + "uses_subprocess": false + }, + { + "path": "scripts/governance_check.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/init_skill.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/judge_blind_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/lint_skill.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/optimize_description.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/promotion_checker.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_artifact_design_profile.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_baseline_compare.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_context_reports.py", + "has_argparse": false, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_description_drift_history.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_eval_dashboard.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/render_intent_confidence.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_intent_dialogue.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_iteration_directions.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_iteration_ledger.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_output_risk_profile.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_portability_report.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_prompt_quality_profile.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_reference_scan.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_reference_synthesis.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_regression_history.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_review_viewer.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_skill_overview.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_social_preview.py", + "has_argparse": false, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/render_system_model.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/resource_boundary_check.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/run_conformance_suite.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/run_description_optimization_suite.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/run_eval_suite.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/run_output_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/skill_report_charts.py", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/skill_report_metrics.py", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/skill_report_model.py", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/sync_local_install.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + }, + { + "path": "scripts/trigger_eval.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/trust_check.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/validate_skill.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": false + }, + { + "path": "scripts/yao.py", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_subprocess": true + } + ], + "dependencies": { + "present": [ + "requirements-ci.txt" + ], + "pinned": [ + "PyYAML==6.0.3" + ], + "unpinned": [] + }, + "trust_metadata": { + "source_tier": "local", + "remote_inline_execution": "forbid", + "remote_metadata_policy": "allow-metadata-only" + }, + "artifacts": { + "json": "reports/security_trust_report.json", + "markdown": "reports/security_trust_report.md" + } +} diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md new file mode 100644 index 00000000..43089a59 --- /dev/null +++ b/reports/security_trust_report.md @@ -0,0 +1,78 @@ +# Security Trust Report + +- OK: `True` +- Scanned files: `124` +- Scripts: `49` +- Secret findings: `0` +- Network-capable scripts: `2` +- Interactive scripts: `0` +- Package SHA256: `d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac` + +## Failures + +- None + +## Warnings + +- Scripts without argparse/help surface: scripts/render_context_reports.py, scripts/render_social_preview.py, scripts/skill_report_charts.py, scripts/skill_report_metrics.py, scripts/skill_report_model.py +- Network-capable scripts require bounded host policy: scripts/check_update.py, scripts/github_benchmark_scan.py + +## Dependency Evidence + +- Files: `requirements-ci.txt` +- Pinned entries: `1` +- Unpinned entries: `0` + +## Script Surface + +| Script | Argparse | Main Guard | Input | Network | Subprocess | +| --- | --- | --- | --- | --- | --- | +| scripts/build_confusion_matrix.py | True | True | False | False | False | +| scripts/build_skill_atlas.py | True | True | False | False | False | +| scripts/check_update.py | True | True | False | True | False | +| scripts/ci_test.py | True | True | False | False | True | +| scripts/collect_feedback.py | True | True | False | False | False | +| scripts/context_sizer.py | True | True | False | False | False | +| scripts/create_iteration_snapshot.py | True | True | False | False | False | +| scripts/cross_packager.py | True | True | False | False | False | +| scripts/diff_eval.py | True | True | False | False | False | +| scripts/export_skill_ir.py | True | True | False | False | False | +| scripts/github_benchmark_scan.py | True | True | False | True | False | +| scripts/governance_check.py | True | True | False | False | False | +| scripts/init_skill.py | True | True | False | False | False | +| scripts/judge_blind_eval.py | True | True | False | False | False | +| scripts/lint_skill.py | True | True | False | False | False | +| scripts/optimize_description.py | True | True | False | False | False | +| scripts/promotion_checker.py | True | True | False | False | False | +| scripts/render_artifact_design_profile.py | True | True | False | False | False | +| scripts/render_baseline_compare.py | True | True | False | False | False | +| scripts/render_context_reports.py | False | True | False | False | False | +| scripts/render_description_drift_history.py | True | True | False | False | False | +| scripts/render_eval_dashboard.py | True | True | False | False | True | +| scripts/render_intent_confidence.py | True | True | False | False | False | +| scripts/render_intent_dialogue.py | True | True | False | False | False | +| scripts/render_iteration_directions.py | True | True | False | False | False | +| scripts/render_iteration_ledger.py | True | True | False | False | False | +| scripts/render_output_risk_profile.py | True | True | False | False | False | +| scripts/render_portability_report.py | True | True | False | False | False | +| scripts/render_prompt_quality_profile.py | True | True | False | False | False | +| scripts/render_reference_scan.py | True | True | False | False | False | +| scripts/render_reference_synthesis.py | True | True | False | False | False | +| scripts/render_regression_history.py | True | True | False | False | False | +| scripts/render_review_viewer.py | True | True | False | False | False | +| scripts/render_skill_overview.py | True | True | False | False | False | +| scripts/render_social_preview.py | False | True | False | False | False | +| scripts/render_system_model.py | True | True | False | False | False | +| scripts/resource_boundary_check.py | True | True | False | False | False | +| scripts/run_conformance_suite.py | True | True | False | False | False | +| scripts/run_description_optimization_suite.py | True | True | False | False | False | +| scripts/run_eval_suite.py | True | True | False | False | True | +| scripts/run_output_eval.py | True | True | False | False | False | +| scripts/skill_report_charts.py | False | False | False | False | False | +| scripts/skill_report_metrics.py | False | False | False | False | False | +| scripts/skill_report_model.py | False | False | False | False | False | +| scripts/sync_local_install.py | True | True | False | False | True | +| scripts/trigger_eval.py | True | True | False | False | False | +| scripts/trust_check.py | True | True | False | False | False | +| scripts/validate_skill.py | True | True | False | False | False | +| scripts/yao.py | True | True | False | False | True | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md new file mode 100644 index 00000000..ffa5f5b8 --- /dev/null +++ b/reports/skill-os-2-review.md @@ -0,0 +1,83 @@ +# Skill OS 2.0 Review + +Review date: 2026-06-13 +Scope: Yao Meta Skill against the user-provided Skill OS 2.0 upgrade plan. + +## Verdict + +Yao Meta Skill is no longer only a Meta Skill factory. The current working tree now has the first verifiable Skill OS foundation: + +- Skill IR v0 for platform-neutral contract capture. +- Output Eval Lab v0 for with-skill vs baseline assertion grading. +- Runtime Conformance v0 for target-consumption checks. +- Trust/Security v0 for secret, dependency, script, trust metadata, and package-integrity checks. +- Skill Atlas v0 for portfolio catalog, route overlap, stale ownership, dependency signals, and no-route opportunities. +- Bilingual Skill Overview v2 that includes these evidence surfaces. + +This is still not the final world-class state. Registry, compiler refactor, telemetry, and Review Studio 2.0 remain open. + +## Coverage Matrix + +| 2.0 Area | Current Evidence | Status | +| --- | --- | --- | +| Skill IR | `skill-ir/schema.json`, `skill-ir/examples/yao-meta-skill.json`, `scripts/export_skill_ir.py` | v0 landed | +| Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `reports/output_quality_scorecard.md` | v0 landed | +| Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed | +| Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | +| Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | +| Review Studio 2.0 | Current `reports/skill-overview.html`, `reports/review-viewer.html`, and `reports/skill_atlas.html` are separate | partial | +| Skill Atlas | `scripts/build_skill_atlas.py`, `skill_atlas/catalog.json`, `skill_atlas/route_overlap_matrix.csv`, `reports/skill_atlas.html` | v0 landed | +| Registry & Distribution | Existing packager, no registry audit/package schema yet | partial | +| Telemetry & Drift | Regression history exists, no adoption or activation telemetry yet | partial | +| Compiler from IR | Packager still reads source metadata directly | missing | + +## Top Findings + +### 1. Compiler path is not IR-first yet + +The new IR captures the capability contract, but `scripts/cross_packager.py` still builds target metadata directly from `SKILL.md` and `agents/interface.yaml`. + +Next move: add `scripts/compile_skill.py` or refactor packager to consume IR for core fields, then assert semantic parity. + +### 2. Output eval is useful but too small + +The v0 cases prove the runner and scorecard work, but the set is only three static cases. The 2.0 plan calls for library/governed coverage with at least five cases, including boundary, near-neighbor, and real file/output constraints. + +Next move: add holdout output cases and one file-backed fixture case. + +### 3. Review Studio is still split + +The overview report is strong, but the reviewer still has to inspect multiple pages for output eval, conformance, trust, and release evidence. + +Next move: add a Review Studio 2.0 section or page that aggregates trigger, output, conformance, trust, and release status into one blocking-issue surface. + +### 4. Multi-skill operation now has v0 coverage, but no telemetry + +The new Skill Atlas can scan a workspace and report catalog, route overlap, dependency graph, stale skill, missing owner/review metadata, and no-route opportunities. It is still static analysis and does not yet include adoption or activation telemetry. + +Next move: connect telemetry and failure history so Atlas can rank stale or conflicting skills by real usage impact. + +### 5. Trust report is structural, not full security review + +Trust v0 blocks obvious secrets and remote inline execution, but it does not yet execute script `--help`, inspect package archive hashes, or enforce per-target permissions. + +Next move: add stricter governed-mode gates and package hash verification after registry format lands. + +## Current Gate Evidence + +| Gate | Current Result | +| --- | --- | +| Output Eval | `3` cases, with-skill pass rate `100`, baseline pass rate `0` | +| Runtime Conformance | `5 / 5` targets passing | +| Trust | `0` secret findings, `1` pinned dependency file, `2` network-capable scripts flagged as warnings | +| Skill Atlas | local workspace scan generated catalog, route-overlap matrix, dependency graph, stale report, owner gaps, and HTML overview | +| Context Budget | initial load remains under the production budget | +| CI | `make ci-test` passed after adding v0 gates | + +## Next Highest-Leverage Moves + +1. Refactor packaging toward IR-first compiler behavior. +2. Add Review Studio 2.0 as a single reviewer surface. +3. Expand Output Eval Lab from static smoke cases to holdout and file-backed cases. +4. Add registry package schema, package hash, and upgrade audit. +5. Connect Skill Atlas with telemetry and drift history. diff --git a/reports/skill-overview.html b/reports/skill-overview.html index a6e5ac79..ad83bfa5 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -606,7 +606,7 @@ -
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分75已生成 6 / 8 类报告证据。已生成 6 / 8 类报告证据。
上下文成本42入口约 362 个词/字，references 约 10018 个词/字。入口约 362 个词/字，references 约 10018 个词/字。
+
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
上下文成本42入口约 366 个词/字，references 约 11029 个词/字。入口约 366 个词/字，references 约 11029 个词/字。
@@ -618,7 +618,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-overview.html
+
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/output_quality_scorecard.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/skill_atlas.html, reports/skill-overview.html
交付流程输入材料Skill 包体可复用能力
交付流程把用户输入、生成的包体和可复用能力放在一条线上。
@@ -634,14 +634,14 @@
-
评分雷达完整度触发清晰证据充分可维护性可迁移性
评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。
+
评分雷达完整度触发清晰证据充分可维护性可迁移性
评分雷达展示结构完整度、触发边界、证据、维护和迁移的相对强弱。

指标判读Reading

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 可用Usable证据充分75已生成 6 / 8 类报告证据。已生成 6 / 8 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 362 个词/字。SKILL.md 约 362 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 362 个词/字，references 约 10018 个词/字。入口约 362 个词/字，references 约 10018 个词/字。
+
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 366 个词/字。SKILL.md 约 366 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 366 个词/字，references 约 11029 个词/字。入口约 366 个词/字，references 约 11029 个词/字。
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分75
  • 已生成 6 / 8 类报告证据。已生成 6 / 8 类报告证据。
  • intent-confidence.json 已存在。intent-confidence.json 已存在。
  • reference-synthesis.json 已存在。reference-synthesis.json 已存在。
可维护性100
  • SKILL.md 约 362 个词/字。SKILL.md 约 362 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 362 个词/字，references 约 10018 个词/字。入口约 362 个词/字，references 约 10018 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • intent-dialogue.json 已存在。intent-dialogue.json 已存在。
可维护性100
  • SKILL.md 约 366 个词/字。SKILL.md 约 366 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 366 个词/字，references 约 11029 个词/字。入口约 366 个词/字，references 约 11029 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
@@ -717,7 +717,7 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已打包 agents/interface.yaml，便于后续做跨平台适配。Portable interface metadata is packaged for later adapter-based export.保留并复用Keep
强项Strength长指导被拆到 references 中，入口文件可以保持轻量。Extended guidance is separated into references so the entrypoint can stay compact.保留并复用Keep
缺口Gap上下文成本需要补强：入口约 362 个词/字，references 约 10018 个词/字。上下文成本需要补强：入口约 362 个词/字，references 约 10018 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 366 个词/字，references 约 11029 个词/字。上下文成本需要补强：入口约 366 个词/字，references 约 11029 个词/字。纳入下一轮修复Fix next
@@ -730,11 +730,11 @@

提前暴露误触发、漂移、证据不足和迁移风险。Surfaces trigger, drift, evidence, and portability risks before the package grows.

-
风险热力000110210发生概率影响程度
风险热力图用影响程度和发生概率标出当前治理重点。
+
风险热力000200300发生概率影响程度
风险热力图用影响程度和发生概率标出当前治理重点。
- +
风险Risk信号Signal应对Response
误触发风险误触发风险frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
输出漂移风险输出漂移风险已生成 6 / 8 类报告证据。已生成 6 / 8 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
证据不足风险证据不足风险已生成 6 / 8 类报告证据。已生成 6 / 8 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
包体膨胀风险包体膨胀风险SKILL.md 约 362 个词/字。SKILL.md 约 362 个词/字。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
跨平台迁移风险跨平台迁移风险agents/interface.yaml 已存在。agents/interface.yaml 已存在。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
误触发风险误触发风险frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
输出漂移风险输出漂移风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
证据不足风险证据不足风险已生成 13 / 13 类报告证据。已生成 13 / 13 类报告证据。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
包体膨胀风险包体膨胀风险SKILL.md 约 366 个词/字。SKILL.md 约 366 个词/字。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
跨平台迁移风险跨平台迁移风险agents/interface.yaml 已存在。agents/interface.yaml 已存在。先补证据和边界，再增加包体复杂度。先补证据和边界，再增加包体复杂度。
@@ -748,7 +748,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布235项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布250项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
@@ -764,8 +764,8 @@

把下一步升级收束为少数高价值动作。Keeps next iteration moves focused and actionable.

-
迭代时间下一步 1Harden portabilit…下一步 2Create an iterati…
迭代时间线把下一步升级收束成少数可执行动作。
-
下一步 1Next 1

加固跨环境语义Harden portability semantics

这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。The skill already signals reuse across environments, so contract clarity matters early.

  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Confirm activation mode, execution context, and trust assumptions.
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Add or review degradation strategy for non-native targets.
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Package the skill once to verify adapter expectations.

跨环境复用更安全，目标漂移更少。Safer cross-environment reuse with less target drift.

下一步 2Next 2

建立迭代证据回路Create an iteration evidence loop

第一版之后，包体应该能说明改了什么以及为什么改。The package should show what changed and why after the first draft.

  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Generate the HTML skill report and keep it aligned with the package.
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Record reference scan choices and non-goals.
  • 原始说明可切换到英文查看；默认中文报告保留结论与结构说明。Capture the next iteration choice explicitly before adding more files.

让下一位作者或评审者更容易接手。A clearer path for the next author or reviewer.

+
迭代时间下一步 1Borrow one proven…下一步 2Harden portabilit…下一步 3Create an iterati…
迭代时间线把下一步升级收束成少数可执行动作。
+
下一步 1Next 1

有选择地借鉴一个成熟模式Borrow one proven pattern on purpose

已经有公开 benchmark 对象，下一步应主动选择一个模式借鉴，而不是松散吸收所有做法。You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.

  • 阅读 `obra/superpowers` 中最值得借鉴的模式。Read the strongest pattern from obra/superpowers.
  • 先判断要借鉴的是方法、结构、执行方式还是可迁移性，并且第一轮只借鉴其中一个。Decide whether to borrow method, structure, execution, or portability, but only one of them first.
  • 记录本轮不借鉴的内容，避免包体过重。Record what you will not borrow so the package stays light.

包体形态更清晰，也减少偶然过度设计。A cleaner package shape with less accidental over-design.

下一步 2Next 2

加固跨环境语义Harden portability semantics

这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。The skill already signals reuse across environments, so contract clarity matters early.

  • 确认激活模式、执行上下文和信任假设。Confirm activation mode, execution context, and trust assumptions.
  • 补充或复核非原生目标端的降级策略。Add or review degradation strategy for non-native targets.
  • 至少打包一次 Skill，用来验证 adapter 预期。Package the skill once to verify adapter expectations.

跨环境复用更安全，目标漂移更少。Safer cross-environment reuse with less target drift.

下一步 3Next 3

建立迭代证据回路Create an iteration evidence loop

第一版之后，包体应该能说明改了什么以及为什么改。The package should show what changed and why after the first draft.

  • 生成 HTML Skill 报告，并保持它与包体内容一致。Generate the HTML skill report and keep it aligned with the package.
  • 记录参考扫描的取舍和非目标。Record reference scan choices and non-goals.
  • 在继续增加文件前，明确记录下一轮迭代选择。Capture the next iteration choice explicitly before adding more files.

让下一位作者或评审者更容易接手。A clearer path for the next author or reviewer.

diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 9650d3d0..873d6459 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -11,6 +11,11 @@ "deliverables": [ "SKILL.md", "agents/interface.yaml", + "reports/skill-ir.json", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/skill_atlas.html", "reports/skill-overview.html" ], "flow": [ @@ -44,20 +49,19 @@ }, "evidence_score": { "label": "证据充分", - "score": 75, + "score": 100, "reasons": [ - "已生成 6 / 8 类报告证据。", - "intent-confidence.json 已存在。", - "reference-synthesis.json 已存在。", - "artifact-design-profile.json 已存在。", - "证据不足：缺少 intent-dialogue.json, output-risk-profile.json。" + "已生成 13 / 13 类报告证据。", + "skill-ir.json 已存在。", + "intent-dialogue.json 已存在。", + "intent-confidence.json 已存在。" ] }, "maintainability_score": { "label": "可维护性", "score": 100, "reasons": [ - "SKILL.md 约 362 个词/字。", + "SKILL.md 约 366 个词/字。", "入口文件保持克制，可维护性较好。", "references/ 已承载扩展指导。", "scripts/ 已承载确定性逻辑。", @@ -78,7 +82,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 362 个词/字，references 约 10018 个词/字。", + "入口约 366 个词/字，references 约 11029 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -175,13 +179,14 @@ "quality_review": { "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", - "已打包 agents/interface.yaml，便于后续做跨平台适配。", - "长指导被拆到 references 中，入口文件可以保持轻量。", - "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。", - "包内包含可随 Skill 迁移的质量门禁或触发检查。" + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", + "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", + "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" ], "gaps": [ - "上下文成本需要补强：入口约 362 个词/字，references 约 10018 个词/字。" + "上下文成本需要补强：入口约 366 个词/字，references 约 11029 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -189,12 +194,12 @@ "每次升级后重新生成报告并检查分数原因。" ], "artifact_design": { - "design_system": "content-led editorial", + "design_system": "evidence frame", "highlights": [ - "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", - "Keep the first screen useful without requiring the reader to parse every detail.", - "Use tables only for comparisons; move explanations below the table.", - "Keep source notes readable without flooding the body with markers." + "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding." ] }, "prompt_quality": { @@ -208,11 +213,11 @@ }, "system_model": { "stability": { - "score": 78, + "score": 86, "band": "stable-first-pass" }, "highlights": [ - "Stability: stable-first-pass (78/100).", + "Stability: stable-first-pass (86/100).", "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." @@ -231,22 +236,22 @@ { "name": "输出漂移风险", "impact": 2, - "probability": 2, - "signal": "已生成 6 / 8 类报告证据。", + "probability": 1, + "signal": "已生成 13 / 13 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "证据不足风险", "impact": 3, - "probability": 2, - "signal": "已生成 6 / 8 类报告证据。", + "probability": 1, + "signal": "已生成 13 / 13 类报告证据。", "response": "先补证据和边界，再增加包体复杂度。" }, { "name": "包体膨胀风险", "impact": 2, "probability": 1, - "signal": "SKILL.md 约 362 个词/字。", + "signal": "SKILL.md 约 366 个词/字。", "response": "先补证据和边界，再增加包体复杂度。" }, { @@ -257,7 +262,121 @@ "response": "先补证据和边界，再增加包体复杂度。" } ], - "risk_families": [], + "risk_families": [ + { + "key": "markdown_readability", + "label": "Markdown readability", + "matched_keywords": [ + "md", + "report", + "doc" + ], + "score": 3, + "risks": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Mixed heading levels can reduce scanability." + ], + "constraints": [ + "Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.", + "Keep table cells short and move explanations below the table.", + "Use heading levels consistently and keep each section anchored to a user-facing outcome." + ], + "self_repair": [ + "Preview whether each table still reads well when columns are narrow.", + "Convert any table with paragraph-length cells into bullets or cards." + ] + }, + { + "key": "citation_clutter", + "label": "Citation and footnote clutter", + "matched_keywords": [ + "source", + "reference" + ], + "score": 2, + "risks": [ + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Source notes may become more prominent than the tutorial itself." + ], + "constraints": [ + "Attach citations only to claims that need evidence, not to every sentence.", + "Group source notes at the end of a section when inline markers would hurt readability.", + "Keep the main sentence readable without requiring the reader to chase a footnote." + ], + "self_repair": [ + "Remove decorative citations that do not support a material claim.", + "Move repeated source explanations into one compact source note." + ] + }, + { + "key": "visual_capture", + "label": "Screenshot and visual capture", + "matched_keywords": [ + "capture" + ], + "score": 1, + "risks": [ + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Image descriptions can omit the action-relevant region." + ], + "constraints": [ + "Never invent a screenshot; state when visual evidence is missing.", + "Record the source, viewport, and crop intent for any screenshot-dependent output.", + "Describe what the reader should inspect in the image, not just that an image exists." + ], + "self_repair": [ + "Check that every screenshot reference points to a real provided or generated asset.", + "Reword any visual instruction that depends on an unseen screen state." + ] + }, + { + "key": "code_or_command_safety", + "label": "Code and command safety", + "matched_keywords": [ + "script" + ], + "score": 1, + "risks": [ + "Commands can omit environment assumptions, working directory, or rollback notes.", + "Code snippets can look runnable while missing required inputs.", + "Error handling can be either absent or over-engineered." + ], + "constraints": [ + "Name the working directory, required inputs, and expected output for each command.", + "Mark destructive or external side-effect operations explicitly.", + "Prefer the smallest runnable snippet over broad framework code." + ], + "self_repair": [ + "Scan each command for cwd, input, output, and side-effect assumptions.", + "Remove speculative error handling that is not tied to a real failure mode." + ] + }, + { + "key": "tone_and_specificity", + "label": "Tone and specificity", + "matched_keywords": [ + "summary" + ], + "score": 1, + "risks": [ + "Headings and summaries can drift into generic, interchangeable language.", + "The output can sound polished but lose the user's actual taste, audience, or scenario.", + "Strong claims can appear without examples or constraints." + ], + "constraints": [ + "Anchor titles and summaries in the user's audience, object, and concrete outcome.", + "Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.", + "Preserve one distinctive phrase, constraint, or standard from the user's input." + ], + "self_repair": [ + "Replace generic title candidates with scenario-specific alternatives.", + "Delete any polished sentence that could fit almost any project unchanged." + ] + } + ], "human_judgment_boundary": [ "Ask one focused clarification when the real job, output, or exclusion boundary is unclear.", "Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.", @@ -294,28 +413,28 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 23 + "file_count": 28 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 65 + "file_count": 49 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 25 + "file_count": 27 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 118 + "file_count": 142 } ], - "file_count": 235, + "file_count": 250, "folder_count": 4, "distribution": [ { @@ -336,24 +455,34 @@ }, { "label": "references", - "value": 23 + "value": 28 }, { "label": "scripts", - "value": 65 + "value": 49 }, { "label": "evals", - "value": 25 + "value": 27 }, { "label": "reports", - "value": 118 + "value": 142 } ] }, "iteration_roadmap": { "items": [ + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, { "title": "Harden portability semantics", "why": "The skill already signals reuse across environments, so contract clarity matters early.", @@ -453,33 +582,34 @@ "path": "references", "label": "Extended guidance and reusable notes", "kind": "folder", - "file_count": 23 + "file_count": 28 }, { "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 65 + "file_count": 49 }, { "path": "evals", "label": "Trigger and quality checks", "kind": "folder", - "file_count": 25 + "file_count": 27 }, { "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 118 + "file_count": 142 } ], "strengths": [ "触发面保持精简，并锚定在 frontmatter description。", - "已打包 agents/interface.yaml，便于后续做跨平台适配。", - "长指导被拆到 references 中，入口文件可以保持轻量。", - "确定性辅助逻辑放在 scripts 中，而不是藏在提示词里。", - "包内包含可随 Skill 迁移的质量门禁或触发检查。" + "已生成 Skill IR，核心语义可先于平台打包被审查和迁移。", + "已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。", + "已生成 Runtime Conformance Matrix，可审查目标端消费能力。", + "已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。", + "已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。" ], "scenarios": [ "把重复流程沉淀为可复用的 agent skill。", @@ -532,6 +662,16 @@ } ], "roadmap": [ + { + "title": "Borrow one proven pattern on purpose", + "why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.", + "actions": [ + "Read the strongest pattern from obra/superpowers.", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.", + "Record what you will not borrow so the package stays light." + ], + "unlocks": "A cleaner package shape with less accidental over-design." + }, { "title": "Harden portability semantics", "why": "The skill already signals reuse across environments, so contract clarity matters early.", @@ -560,18 +700,57 @@ "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。" ], "benchmark_highlights": [], + "skill_ir": { + "schema_version": "2.0.0", + "target_count": 4, + "trigger_samples": 8, + "output_eval_cases": 2 + }, + "output_quality": { + "case_count": 3, + "baseline_pass_rate": 0.0, + "with_skill_pass_rate": 100.0, + "delta": 100.0, + "regression_count": 0, + "gate_pass": true, + "failure_taxonomy": [] + }, + "runtime_conformance": { + "target_count": 5, + "pass_count": 5, + "fail_count": 0 + }, + "trust_security": { + "scanned_files": 124, + "script_count": 49, + "secret_findings": 0, + "dependency_files": [ + "requirements-ci.txt" + ], + "network_script_count": 2, + "interactive_script_count": 0, + "package_sha256": "d70bd9083059198c4346c743131852b32038e59000d86ed11cae4399df6988ac" + }, + "skill_atlas": { + "skill_count": 12, + "route_collision_count": 5, + "owner_gap_count": 9, + "stale_count": 10, + "shared_resource_count": 0, + "no_route_opportunity_count": 0 + }, "synthesis_highlights": [ "Borrow a first-time operator flow that explains itself before it asks for more structure.", "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", "Borrow the discipline of defining what the skill should not own before growing the package." ], "artifact_design": { - "design_system": "content-led editorial", + "design_system": "evidence frame", "highlights": [ - "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail.", - "Keep the first screen useful without requiring the reader to parse every detail.", - "Use tables only for comparisons; move explanations below the table.", - "Keep source notes readable without flooding the body with markers." + "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.", + "Name the working directory and required inputs for commands.", + "Mark destructive, networked, or external side-effect operations.", + "Prefer the smallest runnable snippet over broad framework scaffolding." ] }, "prompt_quality": { @@ -585,11 +764,11 @@ }, "system_model": { "stability": { - "score": 78, + "score": 86, "band": "stable-first-pass" }, "highlights": [ - "Stability: stable-first-pass (78/100).", + "Stability: stable-first-pass (86/100).", "Owned job: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "Leverage: Clarify the real job boundary — Ask one focused question and update intent context before adding assets.", "Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language." diff --git a/reports/skill_atlas.html b/reports/skill_atlas.html new file mode 100644 index 00000000..a3ebd8ba --- /dev/null +++ b/reports/skill_atlas.html @@ -0,0 +1,46 @@ + + + + + + Skill Atlas + + + +
+

Skill Atlas

+

Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

+
+
Skills12
+
Route Collisions5
+
Owner Gaps9
+
Stale Skills10
+
No-Route Opportunities0
+
+
+

Top Issues

+
  • frontend-review collision
  • frontend-review collision
  • frontend-review collision
  • broken-yaml-skill collision
  • frontend-review duplicate-name
  • frontend-review owner, review_cadence, maturity
  • frontend-review owner, review_cadence, maturity
  • frontend-review owner, review_cadence, maturity
  • note-cleanup owner, review_cadence, maturity
  • frontend-review owner, review_cadence, maturity
  • geo-ranking-article-generator owner, review_cadence, maturity
  • invalid-governance-skill owner, review_cadence
  • broken-yaml-skill owner, review_cadence, maturity
  • broken-skill owner, review_cadence, maturity
  • frontend-review missing updated_at
  • frontend-review missing updated_at
  • frontend-review missing updated_at
  • incident-command-governor review overdue by cadence monthly
  • note-cleanup missing updated_at
  • frontend-review missing updated_at
  • geo-ranking-article-generator missing updated_at
  • invalid-governance-skill missing updated_at
  • broken-yaml-skill missing updated_at
  • broken-skill missing updated_at
+
+
+

Catalog

+
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder
+ + +
NamePathOwnerMaturityReview
yao-meta-skill.Yao Teamgovernedquarterly
release-orchestratorexamples/complex-release-orchestrator/generated-skillYao Teamproductionquarterly
frontend-reviewexamples/evolution-frontend-review/finalmissingunknownmissing
frontend-reviewexamples/evolution-frontend-review/v0missingunknownmissing
frontend-reviewexamples/evolution-frontend-review/v1missingunknownmissing
incident-command-governorexamples/governed-incident-command/generated-skillIncident Operations Teamgovernedmonthly
note-cleanupexamples/simple-note-cleanup/generated-skillmissingunknownmissing
frontend-reviewexamples/team-frontend-review/generated-skillmissingunknownmissing
geo-ranking-article-generatorgeo-ranking-article-generatormissingunknownmissing
invalid-governance-skilltests/fixtures/governance_invalid_manifestmissinggovernedmissing
broken-yaml-skilltests/fixtures/package_invalid_yamlmissingunknownmissing
broken-skilltests/fixtures/package_missing_interface_fieldmissingunknownmissing
+
+ + + diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json new file mode 100644 index 00000000..17b5fdae --- /dev/null +++ b/reports/skill_atlas.json @@ -0,0 +1,758 @@ +{ + "ok": true, + "workspace_root": ".", + "summary": { + "skill_count": 12, + "route_collision_count": 5, + "owner_gap_count": 9, + "stale_count": 10, + "shared_resource_count": 0, + "no_route_opportunity_count": 0 + }, + "catalog": { + "workspace_root": ".", + "generated_at": "2026-06-13", + "skills": [ + { + "name": "yao-meta-skill", + "path": ".", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "owner": "Yao Team", + "version": "1.1.0", + "status": "active", + "maturity": "governed", + "updated_at": "2026-03-31", + "review_cadence": "quarterly", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "resources": [ + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/optimize_description.py", + "scripts/promotion_checker.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_viewer.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/validate_skill.py", + "scripts/yao.py", + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/trust-security-method.md", + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "token_set": [ + "add", + "asked", + "docs", + "evals", + "evaluate", + "existing", + "improve", + "notes", + "or", + "package", + "process", + "prompts", + "refactor", + "repeated", + "reuse", + "team", + "transcripts", + "workflows" + ] + }, + { + "name": "release-orchestrator", + "path": "examples/complex-release-orchestrator/generated-skill", + "description": "Coordinate software release preparation, rollout readiness, migration notes, rollback planning, stakeholder communication, and final go/no-go release packets. Use when asked to prepare a release packet, review release readiness, or turn scattered launch notes into one reusable release workflow. Do not use for generic project planning, one-off announcement drafting, or high-level release theory.", + "owner": "Yao Team", + "version": "1.0.0", + "status": "active", + "maturity": "production", + "updated_at": "2026-03-31", + "review_cadence": "quarterly", + "targets": [], + "resources": [ + "scripts/build_release_packet.py", + "references/migration-template.md", + "references/release-checklist.md", + "references/risk-matrix.md", + "references/stakeholder-comms.md" + ], + "token_set": [ + "announcement", + "asked", + "communication", + "coordinate", + "do", + "drafting", + "final", + "generic", + "go", + "high-level", + "launch", + "migration", + "no-go", + "not", + "notes", + "one", + "one-off", + "or", + "packet", + "packets", + "planning", + "preparation", + "prepare", + "project", + "readiness", + "release", + "review", + "rollback", + "rollout", + "scattered", + "software", + "stakeholder", + "theory", + "workflow" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "a11y", + "accessibility", + "asked", + "code", + "common", + "components", + "forms", + "frontend", + "issues", + "missing", + "or", + "pre-merge", + "quality", + "react", + "regressions", + "review", + "risky", + "security", + "states", + "ui", + "ux" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "description": "Review frontend code before merge.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "before", + "code", + "frontend", + "merge", + "review" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "description": "Review frontend code for accessibility, risky UI security issues, missing loading and error states, and pre-merge regressions. Use when asked to review React, UI, forms, frontend components, or merge-readiness for the frontend.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "accessibility", + "asked", + "code", + "components", + "error", + "forms", + "frontend", + "issues", + "loading", + "merge-readiness", + "missing", + "or", + "pre-merge", + "react", + "regressions", + "review", + "risky", + "security", + "states", + "ui" + ] + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill", + "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "owner": "Incident Operations Team", + "version": "1.0.0", + "status": "active", + "maturity": "governed", + "updated_at": "2026-03-31", + "review_cadence": "monthly", + "targets": [ + "openai", + "claude", + "generic" + ], + "resources": [ + "scripts/build_incident_packet.py", + "references/comms-policy.md", + "references/review-policy.md", + "references/severity-matrix.md" + ], + "token_set": [ + "asked", + "assemble", + "assessment", + "build", + "command", + "communication", + "governed", + "incident", + "or", + "packets", + "review", + "run", + "severity", + "standardize" + ] + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "description": "Clean messy meeting notes into structured markdown summaries. Use when asked to organize meeting notes, extract action items, separate decisions from open questions, or turn rough notes into a clean recap.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "action", + "asked", + "clean", + "decisions", + "extract", + "items", + "markdown", + "meeting", + "messy", + "notes", + "open", + "or", + "organize", + "questions", + "recap", + "rough", + "separate", + "structured", + "summaries" + ] + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [ + "references/checklist.md" + ], + "token_set": [ + "a11y", + "accessibility", + "asked", + "check", + "code", + "components", + "frontend", + "missing", + "or", + "pre-merge", + "react", + "regressions", + "rendering", + "review", + "run", + "security", + "states", + "ui", + "unsafe", + "ux" + ] + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "description": "Create GEO ranking-article packs from brand briefs. Use when asked to research a brand, find GEO or 生成式引擎优化 keywords and five competitors, generate timely titles, write five ~2000-character articles, and save them to a fixed knowledge-base folder with an HTML index. Do not use for generic SEO advice, one-off copy, or writing without competitor research.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [ + "scripts/build_html_index.py", + "references/article-blueprint.md", + "references/feishu-benchmark-patterns.md", + "references/geo-research-playbook.md", + "templates/brand-brief-template.md" + ], + "token_set": [ + "2000-character", + "advice", + "articles", + "asked", + "brand", + "briefs", + "competitor", + "competitors", + "copy", + "do", + "find", + "five", + "fixed", + "folder", + "generate", + "generic", + "geo", + "html", + "index", + "keywords", + "knowledge-base", + "not", + "one-off", + "or", + "packs", + "ranking-article", + "research", + "save", + "seo", + "them", + "timely", + "titles", + "without", + "write", + "writing", + "生成式引擎优化" + ] + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "description": "Test fixture for governance validation.", + "owner": "", + "version": "1.0.0", + "status": "frozen", + "maturity": "governed", + "updated_at": "2026/03/31", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "fixture", + "governance", + "test", + "validation" + ] + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "description": "Broken YAML fixture for packager failure testing.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "broken", + "failure", + "fixture", + "packager", + "testing", + "yaml" + ] + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "description": "Broken fixture for packager failure testing.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "broken", + "failure", + "fixture", + "packager", + "testing" + ] + } + ], + "summary": { + "skill_count": 12, + "route_collision_count": 5, + "owner_gap_count": 9, + "stale_count": 10, + "shared_resource_count": 0, + "no_route_opportunity_count": 0 + } + }, + "route_collisions": [ + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/final", + "path_b": "examples/evolution-frontend-review/v1", + "score": 0.708, + "status": "collision" + }, + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/final", + "path_b": "examples/team-frontend-review/generated-skill", + "score": 0.64, + "status": "collision" + }, + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/v1", + "path_b": "examples/team-frontend-review/generated-skill", + "score": 0.538, + "status": "collision" + }, + { + "skill_a": "broken-yaml-skill", + "skill_b": "broken-skill", + "path_a": "tests/fixtures/package_invalid_yaml", + "path_b": "tests/fixtures/package_missing_interface_field", + "score": 0.833, + "status": "collision" + }, + { + "skill_a": "frontend-review", + "skill_b": "frontend-review", + "path_a": "examples/evolution-frontend-review/final", + "path_b": "examples/evolution-frontend-review/v0", + "score": 1.0, + "status": "duplicate-name" + } + ], + "dependency_graph": { + "nodes": [ + { + "name": "yao-meta-skill", + "path": "." + }, + { + "name": "release-orchestrator", + "path": "examples/complex-release-orchestrator/generated-skill" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1" + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill" + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill" + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill" + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator" + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest" + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml" + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field" + } + ], + "shared_resources": [] + }, + "stale_skills": [ + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "reason": "missing updated_at" + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill", + "reason": "review overdue by cadence monthly", + "age_days": 74, + "allowed_days": 31 + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "reason": "missing updated_at" + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "reason": "missing updated_at" + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "reason": "missing updated_at" + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "reason": "missing updated_at" + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "reason": "missing updated_at" + } + ], + "owner_review_gaps": [ + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "missing": [ + "owner", + "review_cadence" + ] + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + } + ], + "no_route_opportunities": [], + "artifacts": { + "catalog": "skill_atlas/catalog.json", + "route_overlap_matrix": "skill_atlas/route_overlap_matrix.csv", + "dependency_graph": "skill_atlas/dependency_graph.json", + "stale_skills": "skill_atlas/stale_skills.json", + "owner_review_gaps": "skill_atlas/owner_review_gaps.json", + "no_route_opportunities": "skill_atlas/no_route_opportunities.json", + "report_json": "reports/skill_atlas.json", + "report_html": "reports/skill_atlas.html" + } +} diff --git a/reports/system-model.json b/reports/system-model.json index fdd904df..40e537b0 100644 --- a/reports/system-model.json +++ b/reports/system-model.json @@ -3,7 +3,7 @@ "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", "systems_doctrine": "Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight.", "stability": { - "score": 78, + "score": 86, "band": "stable-first-pass" }, "boundary_map": { @@ -41,8 +41,8 @@ "Borrow a first-time operator flow that explains itself before it asks for more structure.", "Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed.", "Borrow the discipline of defining what the skill should not own before growing the package.", - "Do not mimic product polish that adds UI bulk without improving clarity.", - "Do not create experimental overhead that exceeds the skill's real risk tier." + "Borrow the way it turns a messy workflow into a repeatable operating path.", + "Borrow the clear execution entrypoints and command structure." ] }, { @@ -50,7 +50,13 @@ "signal": "Generated output may fail in recurring domain-specific ways.", "response": "Apply predicted output-risk families as self-repair checks before final output.", "evidence": "reports/output-risk-profile.md", - "current_risk_families": [] + "current_risk_families": [ + "Markdown readability", + "Citation and footnote clutter", + "Screenshot and visual capture", + "Code and command safety", + "Tone and specificity" + ] }, { "name": "Reviewer feedback loop", @@ -77,7 +83,13 @@ "watch_signal": "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", "countermeasure": "Refresh output-risk and artifact-design profiles, then add one self-repair check.", "cadence": "after the first 3-5 real uses", - "risk_families": [] + "risk_families": [ + "Markdown readability", + "Citation and footnote clutter", + "Screenshot and visual capture", + "Code and command safety", + "Tone and specificity" + ] }, { "name": "Reference drift", @@ -107,7 +119,13 @@ "family": "Output degradation", "symptom": "The result is structurally correct but generic, cluttered, or weakly matched to the user's domain.", "repair": "Use output-risk families as pre-final checks.", - "current_risk_families": [] + "current_risk_families": [ + "Markdown readability", + "Citation and footnote clutter", + "Screenshot and visual capture", + "Code and command safety", + "Tone and specificity" + ] }, { "family": "Prompt-behavior mismatch", @@ -131,6 +149,12 @@ "why": "The description is the highest-leverage routing surface.", "move": "Name the recurring job, expected input, output, and strongest non-goal in compact language." }, + { + "rank": 3, + "point": "Install output self-repair checks", + "why": "The likely failure families are: Markdown readability, Citation and footnote clutter, Screenshot and visual capture.", + "move": "Add only the checks that prevent recurring output mistakes." + }, { "rank": 4, "point": "Borrow one pattern, not a whole product", diff --git a/reports/system-model.md b/reports/system-model.md index c4b844bd..85de725a 100644 --- a/reports/system-model.md +++ b/reports/system-model.md @@ -2,7 +2,7 @@ Skill: `yao-meta-skill` -- Stability score: `78/100` +- Stability score: `86/100` - Stability band: `stable-first-pass` - Doctrine: Structure drives behavior: improve the boundary, feedback loops, drift watch, and leverage points before adding weight. @@ -38,14 +38,20 @@ Skill: `yao-meta-skill` - Borrow a first-time operator flow that explains itself before it asks for more structure. - Borrow a small hypothesis-test-learn loop so the first revision is evidence-backed. - Borrow the discipline of defining what the skill should not own before growing the package. - - Do not mimic product polish that adds UI bulk without improving clarity. - - Do not create experimental overhead that exceeds the skill's real risk tier. + - Borrow the way it turns a messy workflow into a repeatable operating path. + - Borrow the clear execution entrypoints and command structure. ### Output quality loop - Signal: Generated output may fail in recurring domain-specific ways. - Response: Apply predicted output-risk families as self-repair checks before final output. - Evidence: reports/output-risk-profile.md +- Current risk families: + - Markdown readability + - Citation and footnote clutter + - Screenshot and visual capture + - Code and command safety + - Tone and specificity ### Reviewer feedback loop @@ -72,6 +78,12 @@ Skill: `yao-meta-skill` - Watch signal: Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain. - Countermeasure: Refresh output-risk and artifact-design profiles, then add one self-repair check. - Cadence: after the first 3-5 real uses +- Risk families: + - Markdown readability + - Citation and footnote clutter + - Screenshot and visual capture + - Code and command safety + - Tone and specificity ### Reference drift @@ -101,6 +113,12 @@ Skill: `yao-meta-skill` - Symptom: The result is structurally correct but generic, cluttered, or weakly matched to the user's domain. - Repair: Use output-risk families as pre-final checks. +- Current Risk Families: + - Markdown readability + - Citation and footnote clutter + - Screenshot and visual capture + - Code and command safety + - Tone and specificity ### Prompt-behavior mismatch @@ -121,6 +139,11 @@ Skill: `yao-meta-skill` - Why: The description is the highest-leverage routing surface. - Move: Name the recurring job, expected input, output, and strongest non-goal in compact language. +### 3. Install output self-repair checks + +- Why: The likely failure families are: Markdown readability, Citation and footnote clutter, Screenshot and visual capture. +- Move: Add only the checks that prevent recurring output mistakes. + ### 4. Borrow one pattern, not a whole product - Why: External references improve quality when reduced to structure, not copied as surface style. diff --git a/runtime/conformance/README.md b/runtime/conformance/README.md new file mode 100644 index 00000000..0b909c53 --- /dev/null +++ b/runtime/conformance/README.md @@ -0,0 +1,19 @@ +# Runtime Conformance + +Runtime Conformance verifies that a skill package can be consumed by target clients without silently losing its core contract. + +The v0 suite checks: + +- frontmatter `name` and `description` +- description length for Agent Skills / VS Code style clients +- `manifest.json` governance metadata +- `agents/interface.yaml` compatibility metadata +- Skill IR semantic alignment +- target degradation notes +- relative resource paths in IR + +Run: + +```bash +python3 scripts/run_conformance_suite.py . +``` diff --git a/runtime/conformance/schema.json b/runtime/conformance/schema.json new file mode 100644 index 00000000..61f50c68 --- /dev/null +++ b/runtime/conformance/schema.json @@ -0,0 +1,56 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.example/runtime-conformance-report.schema.json", + "title": "Yao Runtime Conformance Report", + "description": "Runtime target conformance matrix for Skill OS 2.0.", + "type": "object", + "additionalProperties": false, + "required": ["ok", "skill", "targets", "summary", "artifacts"], + "properties": { + "ok": {"type": "boolean"}, + "skill": {"type": "string"}, + "targets": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["target", "status", "checks", "failures", "warnings"], + "properties": { + "target": {"type": "string"}, + "status": {"enum": ["pass", "fail"]}, + "checks": { + "type": "array", + "items": {"type": "string"} + }, + "failures": { + "type": "array", + "items": {"type": "string"} + }, + "warnings": { + "type": "array", + "items": {"type": "string"} + } + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": ["target_count", "pass_count", "fail_count"], + "properties": { + "target_count": {"type": "integer"}, + "pass_count": {"type": "integer"}, + "fail_count": {"type": "integer"} + } + }, + "artifacts": { + "type": "object", + "additionalProperties": false, + "required": ["json", "markdown"], + "properties": { + "json": {"type": "string"}, + "markdown": {"type": "string"} + } + } + } +} diff --git a/scripts/build_skill_atlas.py b/scripts/build_skill_atlas.py new file mode 100644 index 00000000..dc80bc37 --- /dev/null +++ b/scripts/build_skill_atlas.py @@ -0,0 +1,444 @@ +#!/usr/bin/env python3 +import argparse +import csv +import html +import json +import re +from collections import Counter, defaultdict +from datetime import date, datetime +from io import StringIO +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent +IGNORE_PARTS = { + ".git", + "__pycache__", + "dist", + ".previews", + "node_modules", + ".venv", + "venv", +} +STOPWORDS = { + "a", + "an", + "and", + "the", + "to", + "for", + "from", + "with", + "into", + "skill", + "skills", + "agent", + "reusable", + "use", + "when", + "create", + "turn", +} +CADENCE_DAYS = { + "monthly": 31, + "quarterly": 100, + "semiannual": 200, + "annual": 370, + "per-release": 120, +} + + +def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]: + text = path.read_text(encoding="utf-8", errors="replace") + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter_text = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + payload = yaml.safe_load(frontmatter_text) or {} + return payload if isinstance(payload, dict) else {}, body + payload = {} + for line in frontmatter_text.splitlines(): + if ":" in line: + key, value = line.split(":", 1) + payload[key.strip()] = value.strip().strip('"') + return payload, body + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def should_skip(path: Path, root: Path) -> bool: + try: + rel = path.relative_to(root) + except ValueError: + return True + return any(part in IGNORE_PARTS for part in rel.parts) + + +def find_skill_dirs(workspace_root: Path) -> list[Path]: + workspace_root = workspace_root.resolve() + skill_dirs = [] + for skill_md in sorted(workspace_root.rglob("SKILL.md")): + if should_skip(skill_md, workspace_root): + continue + skill_dirs.append(skill_md.parent) + return skill_dirs + + +def tokens(text: str) -> set[str]: + raw = re.findall(r"[a-zA-Z0-9_\-\u4e00-\u9fff]{2,}", text.casefold()) + return {item for item in raw if item not in STOPWORDS} + + +def jaccard(left: set[str], right: set[str]) -> float: + if not left or not right: + return 0.0 + return len(left & right) / len(left | right) + + +def safe_rel(root: Path, path: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def resource_names(skill_dir: Path) -> list[str]: + names = [] + for folder in ("scripts", "references", "assets", "templates"): + target = skill_dir / folder + if not target.exists(): + continue + for path in sorted(target.rglob("*")): + if any(part in IGNORE_PARTS for part in path.relative_to(skill_dir).parts): + continue + if path.suffix in {".pyc", ".pyo"}: + continue + if path.is_file() and not path.is_symlink(): + names.append(f"{folder}/{path.name}") + return names + + +def collect_skill(workspace_root: Path, skill_dir: Path) -> dict[str, Any]: + frontmatter, _ = parse_frontmatter(skill_dir / "SKILL.md") + manifest = load_json(skill_dir / "manifest.json") + name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name) + description = str(frontmatter.get("description") or "") + targets = manifest.get("target_platforms", []) + return { + "name": name, + "path": safe_rel(workspace_root, skill_dir), + "description": description, + "owner": str(manifest.get("owner", "")), + "version": str(manifest.get("version", "")), + "status": str(manifest.get("status", "")), + "maturity": str(manifest.get("maturity_tier", manifest.get("skill_archetype", ""))), + "updated_at": str(manifest.get("updated_at", "")), + "review_cadence": str(manifest.get("review_cadence", "")), + "targets": [str(item) for item in targets] if isinstance(targets, list) else [], + "resources": resource_names(skill_dir), + "token_set": sorted(tokens(description)), + } + + +def route_overlap(skills: list[dict[str, Any]], threshold: float) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + rows = [] + collisions = [] + for i, left in enumerate(skills): + for right in skills[i + 1 :]: + score = round(jaccard(set(left["token_set"]), set(right["token_set"])), 3) + status = "collision" if score >= threshold else "clear" + row = { + "skill_a": left["name"], + "skill_b": right["name"], + "path_a": left["path"], + "path_b": right["path"], + "score": score, + "status": status, + } + rows.append(row) + if status == "collision": + collisions.append(row) + duplicate_names = [ + {"name": name, "paths": [item["path"] for item in skills if item["name"] == name]} + for name, count in Counter(item["name"] for item in skills).items() + if count > 1 + ] + for item in duplicate_names: + collisions.append( + { + "skill_a": item["name"], + "skill_b": item["name"], + "path_a": item["paths"][0], + "path_b": item["paths"][1], + "score": 1.0, + "status": "duplicate-name", + } + ) + return rows, collisions + + +def dependency_graph(skills: list[dict[str, Any]]) -> dict[str, Any]: + by_resource: dict[str, list[str]] = defaultdict(list) + for skill in skills: + for resource in skill.get("resources", []): + by_resource[resource].append(skill["name"]) + shared = [ + {"resource": resource, "skills": sorted(names)} + for resource, names in sorted(by_resource.items()) + if len(set(names)) > 1 + ] + return { + "nodes": [{"name": item["name"], "path": item["path"]} for item in skills], + "shared_resources": shared, + } + + +def parse_date(value: str) -> date | None: + if not value: + return None + try: + return datetime.strptime(value[:10], "%Y-%m-%d").date() + except ValueError: + return None + + +def stale_skills(skills: list[dict[str, Any]], today: date) -> list[dict[str, Any]]: + stale = [] + for skill in skills: + updated = parse_date(skill.get("updated_at", "")) + cadence = skill.get("review_cadence") or "" + allowed_days = CADENCE_DAYS.get(cadence, 120) + if not updated: + stale.append({"name": skill["name"], "path": skill["path"], "reason": "missing updated_at"}) + continue + age = (today - updated).days + if age > allowed_days: + stale.append( + { + "name": skill["name"], + "path": skill["path"], + "reason": f"review overdue by cadence {cadence or 'unspecified'}", + "age_days": age, + "allowed_days": allowed_days, + } + ) + return stale + + +def owner_review_gaps(skills: list[dict[str, Any]]) -> list[dict[str, Any]]: + gaps = [] + for skill in skills: + missing = [] + if not skill.get("owner"): + missing.append("owner") + if not skill.get("review_cadence"): + missing.append("review_cadence") + if not skill.get("maturity"): + missing.append("maturity") + if missing: + gaps.append({"name": skill["name"], "path": skill["path"], "missing": missing}) + return gaps + + +def no_route_opportunities(workspace_root: Path) -> list[dict[str, Any]]: + opportunities = [] + for path in sorted(workspace_root.rglob("failure-cases.md")): + if should_skip(path, workspace_root): + continue + text = path.read_text(encoding="utf-8", errors="replace") + for line in text.splitlines(): + stripped = line.strip() + if not stripped.startswith("-"): + continue + lowered = stripped.casefold() + if "no_route" in lowered or "no route" in lowered or "missed" in lowered or "under-trigger" in lowered: + opportunities.append({"source": safe_rel(workspace_root, path), "note": stripped.lstrip("- ").strip()}) + return opportunities[:50] + + +def write_csv(path: Path, rows: list[dict[str, Any]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fields = ["skill_a", "skill_b", "path_a", "path_b", "score", "status"] + buffer = StringIO() + writer = csv.DictWriter(buffer, fieldnames=fields, lineterminator="\n") + writer.writeheader() + for row in rows: + writer.writerow({field: row.get(field, "") for field in fields}) + path.write_text(buffer.getvalue(), encoding="utf-8") + + +def render_html(payload: dict[str, Any]) -> str: + summary = payload["summary"] + rows = [] + for skill in payload["catalog"]["skills"][:80]: + rows.append( + "" + f"{html.escape(skill['name'])}" + f"{html.escape(skill['path'])}" + f"{html.escape(skill.get('owner') or 'missing')}" + f"{html.escape(skill.get('maturity') or 'unknown')}" + f"{html.escape(skill.get('review_cadence') or 'missing')}" + "" + ) + blockers = payload["route_collisions"][:20] + payload["owner_review_gaps"][:20] + payload["stale_skills"][:20] + blocker_items = "".join( + f"
  • {html.escape(item.get('name', item.get('skill_a', 'issue')))} {html.escape(item.get('reason', item.get('status', ', '.join(item.get('missing', [])))))}
    • " + for item in blockers + ) + return f""" + + + + + Skill Atlas + + + +
    +

    Skill Atlas

    +

    Portfolio-level review for route overlap, stale ownership, shared resources, and no-route opportunities.

    +
    +
    Skills{summary['skill_count']}
    +
    Route Collisions{summary['route_collision_count']}
    +
    Owner Gaps{summary['owner_gap_count']}
    +
    Stale Skills{summary['stale_count']}
    +
    No-Route Opportunities{summary['no_route_opportunity_count']}
    +
    +
    +

    Top Issues

    + +
    +
    +

    Catalog

    + + + {''.join(rows)} +
    NamePathOwnerMaturityReview
    +
    +
    + + +""" + + +def build_atlas(workspace_root: Path, output_dir: Path, report_html: Path, report_json: Path, threshold: float, today: date) -> dict[str, Any]: + workspace_root = workspace_root.resolve() + skill_dirs = find_skill_dirs(workspace_root) + skills = [collect_skill(workspace_root, skill_dir) for skill_dir in skill_dirs] + overlap_rows, collisions = route_overlap(skills, threshold) + graph = dependency_graph(skills) + stale = stale_skills(skills, today) + owner_gaps = owner_review_gaps(skills) + opportunities = no_route_opportunities(workspace_root) + summary = { + "skill_count": len(skills), + "route_collision_count": len(collisions), + "owner_gap_count": len(owner_gaps), + "stale_count": len(stale), + "shared_resource_count": len(graph["shared_resources"]), + "no_route_opportunity_count": len(opportunities), + } + catalog = { + "workspace_root": display_path(workspace_root), + "generated_at": today.isoformat(), + "skills": skills, + "summary": summary, + } + payload = { + "ok": True, + "workspace_root": display_path(workspace_root), + "summary": summary, + "catalog": catalog, + "route_collisions": collisions, + "dependency_graph": graph, + "stale_skills": stale, + "owner_review_gaps": owner_gaps, + "no_route_opportunities": opportunities, + "artifacts": { + "catalog": display_path(output_dir / "catalog.json"), + "route_overlap_matrix": display_path(output_dir / "route_overlap_matrix.csv"), + "dependency_graph": display_path(output_dir / "dependency_graph.json"), + "stale_skills": display_path(output_dir / "stale_skills.json"), + "owner_review_gaps": display_path(output_dir / "owner_review_gaps.json"), + "no_route_opportunities": display_path(output_dir / "no_route_opportunities.json"), + "report_json": display_path(report_json), + "report_html": display_path(report_html), + }, + } + output_dir.mkdir(parents=True, exist_ok=True) + (output_dir / "catalog.json").write_text(json.dumps(catalog, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + write_csv(output_dir / "route_overlap_matrix.csv", overlap_rows) + (output_dir / "dependency_graph.json").write_text(json.dumps(graph, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output_dir / "stale_skills.json").write_text(json.dumps(stale, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output_dir / "owner_review_gaps.json").write_text(json.dumps(owner_gaps, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + (output_dir / "no_route_opportunities.json").write_text(json.dumps(opportunities, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + report_json.parent.mkdir(parents=True, exist_ok=True) + report_html.parent.mkdir(parents=True, exist_ok=True) + report_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + report_html.write_text(render_html(payload), encoding="utf-8") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build a Skill Atlas for a workspace of agent skills.") + parser.add_argument("--workspace-root", default=".") + parser.add_argument("--output-dir", default=str(ROOT / "skill_atlas")) + parser.add_argument("--report-html", default=str(ROOT / "reports" / "skill_atlas.html")) + parser.add_argument("--report-json", default=str(ROOT / "reports" / "skill_atlas.json")) + parser.add_argument("--overlap-threshold", type=float, default=0.42) + parser.add_argument("--today", default=date.today().isoformat()) + args = parser.parse_args() + today = datetime.strptime(args.today, "%Y-%m-%d").date() + payload = build_atlas( + Path(args.workspace_root).resolve(), + Path(args.output_dir).resolve(), + Path(args.report_html).resolve(), + Path(args.report_json).resolve(), + args.overlap_threshold, + today, + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 0990ce76..52d5c79d 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -33,6 +33,11 @@ DEFAULT_TARGETS = [ "skill-overview-check", "skill-report-metrics-check", "skill-report-charts-check", + "skill-ir-check", + "output-eval-check", + "runtime-conformance-check", + "trust-check", + "skill-atlas-check", "review-viewer-check", "feedback-check", "baseline-compare-check", diff --git a/scripts/export_skill_ir.py b/scripts/export_skill_ir.py new file mode 100644 index 00000000..2f7870e2 --- /dev/null +++ b/scripts/export_skill_ir.py @@ -0,0 +1,347 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from datetime import date +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent +SCHEMA_VERSION = "2.0.0" +IR_SCHEMA = ROOT / "skill-ir" / "schema.json" +KEY_REPORTS = [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json", +] + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def parse_frontmatter(text: str) -> tuple[dict[str, Any], str]: + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter_text = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + payload = yaml.safe_load(frontmatter_text) or {} + return payload if isinstance(payload, dict) else {}, body + data: dict[str, Any] = {} + for line in frontmatter_text.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data, body + + +def parse_sections(body: str) -> dict[str, str]: + sections: dict[str, list[str]] = {"_preamble": []} + current = "_preamble" + for line in body.splitlines(): + if line.startswith("## "): + current = line[3:].strip() + sections[current] = [] + continue + sections.setdefault(current, []).append(line) + return {name: "\n".join(lines).strip() for name, lines in sections.items()} + + +def extract_list_items(text: str, limit: int = 12) -> list[str]: + items = [] + for line in text.splitlines(): + stripped = line.strip() + match = re.match(r"^(?:[-*]|\d+\.)\s+(.*)$", stripped) + if match: + item = match.group(1).strip() + if item: + items.append(item) + if len(items) >= limit: + break + return items + + +def compact_unique(items: list[str], limit: int = 12) -> list[str]: + seen = set() + output = [] + for item in items: + value = str(item).strip() + key = value.lower() + if not value or key in seen: + continue + seen.add(key) + output.append(value) + if len(output) >= limit: + break + return output + + +def trigger_samples(skill_dir: Path, key: str, limit: int = 8) -> list[str]: + payload = load_json(skill_dir / "evals" / "trigger_cases.json") + values = payload.get(key, []) + samples = [] + if isinstance(values, list): + for item in values: + if isinstance(item, dict): + samples.append(str(item.get("text", "")).strip()) + else: + samples.append(str(item).strip()) + return compact_unique(samples, limit=limit) + + +def report_list(skill_dir: Path) -> list[str]: + return [rel for rel in KEY_REPORTS if (skill_dir / rel).exists()] + + +def file_list(skill_dir: Path, folder: str, suffixes: set[str] | None = None, limit: int = 80) -> list[str]: + target = skill_dir / folder + if not target.exists(): + return [] + paths = [] + for path in sorted(target.rglob("*")): + if not path.is_file(): + continue + if suffixes is not None and path.suffix not in suffixes: + continue + paths.append(str(path.relative_to(skill_dir))) + if len(paths) >= limit: + break + return paths + + +def workflow_steps(sections: dict[str, str]) -> list[str]: + for key in ("Compact Workflow", "Workflow", "How It Works", "Runbook", "Quick Start"): + items = extract_list_items(sections.get(key, ""), limit=10) + if items: + return items + return ["Understand the request.", "Execute the main task.", "Validate the result."] + + +def decision_points(sections: dict[str, str], system_model: dict[str, Any]) -> list[str]: + points = [] + for key in ("Router Rules", "Modes", "Output Contract"): + points.extend(extract_list_items(sections.get(key, ""), limit=6)) + boundary = system_model.get("boundary_map", {}) if isinstance(system_model, dict) else {} + points.extend(boundary.get("human_judgment_boundary", []) if isinstance(boundary, dict) else []) + return compact_unique(points, limit=10) + + +def failure_modes(skill_dir: Path, output_risk: dict[str, Any], system_model: dict[str, Any]) -> list[str]: + modes = [] + modes.extend(output_risk.get("top_risks", [])[:6]) + drift = system_model.get("drift_watch", []) if isinstance(system_model, dict) else [] + for item in drift: + if isinstance(item, dict): + modes.append(str(item.get("watch_signal", "") or item.get("risk", ""))) + failure_cases = skill_dir / "evals" / "failure-cases.md" + if failure_cases.exists(): + modes.extend(extract_list_items(failure_cases.read_text(encoding="utf-8"), limit=4)) + return compact_unique(modes, limit=10) + + +def risk_level_from_count(count: int) -> str: + if count >= 5: + return "high" + if count >= 2: + return "medium" + return "low" + + +def build_skill_ir(skill_dir: Path) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") + frontmatter, body = parse_frontmatter(skill_text) + sections = parse_sections(body) + manifest = load_json(skill_dir / "manifest.json") + interface_data = load_yaml(skill_dir / "agents" / "interface.yaml") + intent_context = load_json(skill_dir / "reports" / "intent-context.json") + output_risk = load_json(skill_dir / "reports" / "output-risk-profile.json") + system_model = load_json(skill_dir / "reports" / "system-model.json") + + name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name) + description = str(frontmatter.get("description") or intent_context.get("description") or "") + title = str(interface_data.get("interface", {}).get("display_name") or name.replace("-", " ").title()) + job = str(intent_context.get("job") or system_model.get("boundary_map", {}).get("owned_job") or description) + exclusions = intent_context.get("exclusions", []) + if not exclusions: + exclusions = system_model.get("boundary_map", {}).get("non_goals", []) + + scripts = file_list(skill_dir, "scripts", suffixes={".py", ".sh", ".js", ".ts"}) + references = file_list(skill_dir, "references", suffixes={".md", ".txt", ".json", ".yaml", ".yml"}) + assets = file_list(skill_dir, "assets") + file_list(skill_dir, "templates") + trigger_eval_files = file_list(skill_dir, "evals", suffixes={".json", ".jsonl", ".md"}) + output_eval_files = file_list(skill_dir, "evals/output", suffixes={".json", ".jsonl", ".md"}) + + target_platforms = manifest.get("target_platforms") or interface_data.get("compatibility", {}).get("adapter_targets", []) + maturity = str(manifest.get("maturity_tier") or manifest.get("skill_archetype") or "scaffold") + trust_boundary = "external" if maturity == "governed" else "team" if maturity in {"production", "library"} else "personal" + output_risk_count = len(output_risk.get("risk_families", [])) + execution_risk = "medium" if scripts else "low" + + return { + "schema_version": SCHEMA_VERSION, + "name": name, + "title": title, + "job_to_be_done": job, + "trigger_surface": { + "description": description, + "should_trigger": trigger_samples(skill_dir, "should_trigger") or [description], + "should_not_trigger": compact_unique([*trigger_samples(skill_dir, "should_not_trigger"), *map(str, exclusions)], 10), + "edge_cases": trigger_samples(skill_dir, "near_neighbor"), + }, + "workflow": { + "steps": workflow_steps(sections), + "decision_points": decision_points(sections, system_model), + "failure_modes": failure_modes(skill_dir, output_risk, system_model), + }, + "resources": { + "references": references, + "scripts": scripts, + "assets": assets, + "reports": report_list(skill_dir), + }, + "eval_plan": { + "trigger": trigger_eval_files, + "output": output_eval_files, + "adversarial": file_list(skill_dir, "evals/adversarial", suffixes={".json", ".jsonl", ".md"}), + "baseline": "without_skill", + }, + "risk": { + "output_risk": risk_level_from_count(output_risk_count), + "execution_risk": execution_risk, + "trust_boundary": trust_boundary, + }, + "governance": { + "owner": str(manifest.get("owner", "")), + "maturity": maturity if maturity in {"scaffold", "production", "library", "governed"} else "scaffold", + "review_cadence": str(manifest.get("review_cadence", "")), + "review_due": str(manifest.get("review_due", "")), + }, + "targets": [str(item) for item in target_platforms], + "source_files": compact_unique( + [ + "SKILL.md", + "manifest.json" if (skill_dir / "manifest.json").exists() else "", + "agents/interface.yaml" if (skill_dir / "agents" / "interface.yaml").exists() else "", + "reports/intent-context.json" if (skill_dir / "reports" / "intent-context.json").exists() else "", + "reports/output-risk-profile.json" if (skill_dir / "reports" / "output-risk-profile.json").exists() else "", + "reports/system-model.json" if (skill_dir / "reports" / "system-model.json").exists() else "", + ], + limit=20, + ), + } + + +def validate_ir(payload: dict[str, Any]) -> list[str]: + failures = [] + for key in ("schema_version", "name", "job_to_be_done", "trigger_surface", "workflow", "resources", "eval_plan", "risk", "governance"): + if key not in payload: + failures.append(f"Missing required field: {key}") + if payload.get("schema_version") != SCHEMA_VERSION: + failures.append(f"schema_version must be {SCHEMA_VERSION}") + if not payload.get("trigger_surface", {}).get("description"): + failures.append("trigger_surface.description is required") + if not payload.get("trigger_surface", {}).get("should_trigger"): + failures.append("trigger_surface.should_trigger needs at least one sample") + if not payload.get("workflow", {}).get("steps"): + failures.append("workflow.steps needs at least one step") + if payload.get("risk", {}).get("output_risk") not in {"low", "medium", "high"}: + failures.append("risk.output_risk must be low, medium, or high") + if payload.get("governance", {}).get("maturity") not in {"scaffold", "production", "library", "governed"}: + failures.append("governance.maturity is invalid") + return failures + + +def default_output_path(skill_dir: Path, payload: dict[str, Any]) -> Path: + if skill_dir.resolve() == ROOT.resolve(): + return ROOT / "skill-ir" / "examples" / f"{payload['name']}.json" + return skill_dir / "reports" / "skill-ir.json" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Export a platform-neutral Skill IR document from a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json") + parser.add_argument("--validate-only", action="store_true") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + payload = build_skill_ir(skill_dir) + failures = validate_ir(payload) + output_json = Path(args.output_json).resolve() if args.output_json else default_output_path(skill_dir, payload) + if not args.validate_only: + output_json.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + result = { + "ok": not failures, + "schema": str(IR_SCHEMA), + "skill_dir": str(skill_dir), + "artifacts": {"json": str(output_json)}, + "summary": { + "name": payload.get("name"), + "maturity": payload.get("governance", {}).get("maturity"), + "target_count": len(payload.get("targets", [])), + "trigger_samples": len(payload.get("trigger_surface", {}).get("should_trigger", [])), + "resource_count": sum(len(payload.get("resources", {}).get(key, [])) for key in ("references", "scripts", "assets", "reports")), + }, + "failures": failures, + } + print(json.dumps(result, ensure_ascii=False, indent=2)) + raise SystemExit(0 if result["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/github_benchmark_scan.py b/scripts/github_benchmark_scan.py index 6007482e..70066712 100644 --- a/scripts/github_benchmark_scan.py +++ b/scripts/github_benchmark_scan.py @@ -231,6 +231,11 @@ def cross_repo_insights(repos: list[dict[str, Any]]) -> dict[str, list[str]]: avoid = [] seen_borrow = set() seen_avoid = set() + if not repos: + return { + "borrow": [], + "avoid": [], + } for repo in repos: for item in repo.get("borrow", []): if item not in seen_borrow: @@ -256,16 +261,23 @@ def repo_to_external_reference(repo: dict[str, Any]) -> dict[str, str]: def render_markdown(payload: dict[str, Any]) -> str: + repo_count = len(payload["repositories"]) + next_step = ( + f"Review the {repo_count} benchmark object{'s' if repo_count != 1 else ''} below, " + "then decide whether any of their patterns are worth borrowing into the new skill." + if repo_count + else "No benchmark repositories were collected; adjust the query or attach manual references before treating the scan as evidence." + ) lines = [ "# GitHub Benchmark Scan", "", f"- Query: `{payload['query']}`", f"- Source: `{payload['source']}`", - f"- Top repositories: `{len(payload['repositories'])}`", + f"- Top repositories: `{repo_count}`", "", "## Suggested Next Step", "", - "Review the three benchmark objects below, then decide whether any of their patterns are worth borrowing into the new skill.", + next_step, "", ] @@ -282,7 +294,7 @@ def render_markdown(payload: dict[str, Any]) -> str: ) return "\n".join(lines).strip() + "\n" - lines.extend(["## Top 3 Benchmark Repositories", ""]) + lines.extend([f"## Top {repo_count} Benchmark Repositories", ""]) for repo in payload["repositories"]: lines.extend( [ @@ -359,7 +371,13 @@ def run_github_benchmark_scan( "repositories": repo_summaries, "cross_repo": cross_repo, "external_references": [repo_to_external_reference(repo) for repo in repo_summaries], - "borrow_prompt": "I found three public GitHub projects worth studying. Do you want to borrow any of these patterns for method, structure, execution, or portability?", + "borrow_prompt": ( + f"I found {len(repo_summaries)} public GitHub project" + f"{'s' if len(repo_summaries) != 1 else ''} worth studying. " + "Do you want to borrow any of these patterns for method, structure, execution, or portability?" + if repo_summaries + else "No benchmark suggestions were collected yet. You can retry with a stronger query or add manual references." + ), "warnings": warnings, } except Exception as exc: # pragma: no cover - network failures are environment-dependent diff --git a/scripts/init_skill.py b/scripts/init_skill.py index 19490ce2..2abdaf99 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -5,6 +5,7 @@ import re from datetime import date from pathlib import Path +from export_skill_ir import build_skill_ir, validate_ir from github_benchmark_scan import run_github_benchmark_scan from render_intent_confidence import render_intent_confidence from render_intent_dialogue import render_intent_dialogue @@ -72,8 +73,9 @@ README_TEMPLATE = """# {title} 9. Check `reports/output-risk-profile.md` to see likely output mistakes and self-repair checks. 10. Check `reports/artifact-design-profile.md` to see the intended artifact direction, layout patterns, visual quality gates, and anti-patterns. 11. Check `reports/prompt-quality-profile.md` to see the need model, RTF-to-skill mapping, complexity, and prompt-facing quality matrix. -12. Review `reports/iteration-directions.md` for the three most valuable next moves. -13. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. +12. Review `reports/skill-ir.json` for the platform-neutral Skill IR contract before platform-specific packaging. +13. Review `reports/iteration-directions.md` for the three most valuable next moves. +14. Review `reports/system-model.md` to understand the boundary, feedback loops, drift watch, failure map, and highest-leverage next changes. ## Honest Boundaries @@ -95,6 +97,7 @@ README_TEMPLATE = """# {title} - `reports/artifact-design-profile.md`: artifact-specific design direction, layout patterns, visual quality gates, and anti-patterns - `reports/prompt-quality-profile.md`: prompt-facing need model, RTF mapping, complexity, and quality matrix - `reports/system-model.md`: systems-thinking model for boundary, feedback loops, drift, failure patterns, and leverage points +- `reports/skill-ir.json`: platform-neutral 2.0 Skill IR contract for trigger, workflow, resources, evals, risk, and governance - `reports/skill-overview.html`: white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, a top-right language switch, metrics, SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap - `reports/review-viewer.html`: compact review page for architecture, usage, feedback, and next steps - `reports/iteration-directions.md`: the top three next iteration directions @@ -222,6 +225,24 @@ def build_report_view(artifacts: dict) -> dict: } +def render_skill_ir(root: Path) -> dict: + payload = build_skill_ir(root) + failures = validate_ir(payload) + output_json = root / "reports" / "skill-ir.json" + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + return { + "ok": not failures, + "artifacts": {"json": str(output_json)}, + "summary": { + "name": payload.get("name"), + "maturity": payload.get("governance", {}).get("maturity"), + "target_count": len(payload.get("targets", [])), + "trigger_samples": len(payload.get("trigger_surface", {}).get("should_trigger", [])), + }, + "failures": failures, + } + + def dedupe_references(references: list[dict]) -> list[dict]: seen = set() deduped = [] @@ -303,6 +324,7 @@ def initialize_skill( artifact_design_profile = render_artifact_design_profile(root) prompt_quality_profile = render_prompt_quality_profile(root) system_model = render_system_model(root) + skill_ir = render_skill_ir(root) iteration_directions = render_iteration_directions(root) overview = render_skill_overview(root) review_viewer = render_review_viewer(root) @@ -328,6 +350,7 @@ def initialize_skill( "prompt_quality_profile_json": prompt_quality_profile["artifacts"]["json"], "system_model_md": system_model["artifacts"]["markdown"], "system_model_json": system_model["artifacts"]["json"], + "skill_ir_json": skill_ir["artifacts"]["json"], "iteration_directions_md": iteration_directions["artifacts"]["markdown"], "iteration_directions_json": iteration_directions["artifacts"]["json"], "review_viewer_html": review_viewer["artifacts"]["html"], @@ -346,6 +369,7 @@ def initialize_skill( "intent_confidence": intent_confidence["summary"], "reference_synthesis": reference_synthesis["summary"], "system_model": system_model["summary"], + "skill_ir": skill_ir["summary"], "artifacts": artifacts, "report_view": report_view, } diff --git a/scripts/render_skill_overview.py b/scripts/render_skill_overview.py index e3bebc5d..ff2a404c 100644 --- a/scripts/render_skill_overview.py +++ b/scripts/render_skill_overview.py @@ -22,14 +22,17 @@ TEXT_ZH = { "Write the `description` early and test route quality before expanding the package.": "尽早写出 `description`，先测试路由质量，再扩展包体。", "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.": "只在确有价值时添加 output-risk、artifact-design、prompt-quality 和 system-model 报告。", "Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.": "当你需要把工作流或笔记沉淀成结构精简、触发清晰且带必要 eval 的可复用 Skill 时使用 $yao-meta-skill。", + "Turn rough requests into a compact reusable demo skill.": "把粗糙请求整理成紧凑、可复用的演示 Skill。", "Tighten trigger and exclusions": "收紧触发与排除边界", "Add the first execution asset": "补上第一个执行资产", "Promote from scaffold to production-ready": "从脚手架推进到生产可用", + "Borrow one proven pattern on purpose": "有选择地借鉴一个成熟模式", "Harden portability semantics": "加固跨环境语义", "Create an iteration evidence loop": "建立迭代证据回路", "The package needs clearer near-neighbor exclusions before it grows.": "在继续扩展前，需要先把相邻但不应触发的场景说清楚。", "The package is still mostly prose. Add one asset that removes repeated manual work.": "当前包体仍偏文本说明，应先增加一个能减少重复人工操作的资产。", "The first version exists; the next gain usually comes from adding the smallest useful gates.": "第一版已经存在，下一步收益通常来自补上最小但有效的质量门禁。", + "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.": "已经有公开 benchmark 对象，下一步应主动选择一个模式借鉴，而不是松散吸收所有做法。", "The skill already signals reuse across environments, so contract clarity matters early.": "这个 Skill 已经面向跨环境复用，因此早期就需要把契约语义说清楚。", "The package should show what changed and why after the first draft.": "第一版之后，包体应该能说明改了什么以及为什么改。", "Add 3 to 5 should-trigger and should-not-trigger examples.": "增加 3 到 5 个应触发和不应触发的例子。", @@ -41,9 +44,18 @@ TEXT_ZH = { "Decide whether this skill is personal, team-reused, or library-grade.": "判断这个 Skill 是个人使用、团队复用，还是库级基础能力。", "Add only the gates that match that risk level.": "只添加与风险等级匹配的质量门禁。", "Record lifecycle metadata and review cadence once reuse becomes real.": "一旦进入真实复用，就记录生命周期元数据和评审节奏。", + "Decide whether to borrow method, structure, execution, or portability, but only one of them first.": "先判断要借鉴的是方法、结构、执行方式还是可迁移性，并且第一轮只借鉴其中一个。", + "Record what you will not borrow so the package stays light.": "记录本轮不借鉴的内容，避免包体过重。", + "Confirm activation mode, execution context, and trust assumptions.": "确认激活模式、执行上下文和信任假设。", + "Add or review degradation strategy for non-native targets.": "补充或复核非原生目标端的降级策略。", + "Package the skill once to verify adapter expectations.": "至少打包一次 Skill，用来验证 adapter 预期。", + "Generate the HTML skill report and keep it aligned with the package.": "生成 HTML Skill 报告，并保持它与包体内容一致。", + "Record reference scan choices and non-goals.": "记录参考扫描的取舍和非目标。", + "Capture the next iteration choice explicitly before adding more files.": "在继续增加文件前，明确记录下一轮迭代选择。", "Cleaner routing and fewer accidental activations.": "路由更清晰，误触发更少。", "Stronger execution quality without bloating the entrypoint.": "在不膨胀入口文件的前提下提升执行质量。", "A clearer path from exploratory package to maintained asset.": "更清晰地从探索性包体走向可维护资产。", + "A cleaner package shape with less accidental over-design.": "包体形态更清晰，也减少偶然过度设计。", "Safer cross-environment reuse with less target drift.": "跨环境复用更安全，目标漂移更少。", "A clearer path for the next author or reviewer.": "让下一位作者或评审者更容易接手。", } @@ -106,6 +118,12 @@ def zh_for(text: str) -> str: return "当用户请求与该 Skill 的触发描述匹配时使用。" if value.startswith("用户说出类似需求时："): return "当用户提出与该 Skill 触发描述相近的请求时使用。" + if value.startswith("Use $") and " when you need to " in value: + skill, need = value.removeprefix("Use ").split(" when you need to ", 1) + return f"当你需要{zh_for(need).rstrip('。')}时使用 `{skill}`。" + if value.startswith("Read the strongest pattern from "): + repo = value.removeprefix("Read the strongest pattern from ").rstrip(".") + return f"阅读 `{repo}` 中最值得借鉴的模式。" if value.startswith("Primary prompt task family:"): return "主要提示任务类型已记录在 prompt quality profile 中。" if value.startswith("Complexity:"): diff --git a/scripts/run_conformance_suite.py b/scripts/run_conformance_suite.py new file mode 100644 index 00000000..21ab109a --- /dev/null +++ b/scripts/run_conformance_suite.py @@ -0,0 +1,264 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_TARGETS = ["openai", "claude", "agent-skills", "vscode", "generic"] + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def parse_frontmatter(path: Path) -> tuple[dict[str, Any], str]: + if not path.exists(): + return {}, "" + text = path.read_text(encoding="utf-8") + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter_text = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + payload = yaml.safe_load(frontmatter_text) or {} + return payload if isinstance(payload, dict) else {}, body + data = {} + for line in frontmatter_text.splitlines(): + if ":" in line: + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data, body + + +def find_skill_ir(skill_dir: Path, name: str) -> dict[str, Any]: + direct = load_json(skill_dir / "reports" / "skill-ir.json") + if direct: + return direct + return load_json(skill_dir / "skill-ir" / "examples" / f"{name}.json") + + +def add_check(checks: list[str], failures: list[str], condition: bool, passed: str, failed: str) -> None: + if condition: + checks.append(passed) + else: + failures.append(failed) + + +def relative_resource_check(skill_dir: Path, ir: dict[str, Any]) -> tuple[list[str], list[str]]: + checks = [] + failures = [] + resources = ir.get("resources", {}) if isinstance(ir, dict) else {} + for group in ("references", "scripts", "assets", "reports"): + for raw_path in resources.get(group, []): + rel = Path(str(raw_path)) + if rel.is_absolute(): + failures.append(f"{group} resource is absolute: {raw_path}") + continue + target = (skill_dir / rel).resolve() + try: + target.relative_to(skill_dir.resolve()) + except ValueError: + failures.append(f"{group} resource escapes package: {raw_path}") + continue + if not target.exists(): + failures.append(f"{group} resource is missing: {raw_path}") + else: + checks.append(f"{group} resource resolves: {raw_path}") + return checks, failures + + +def common_evidence(skill_dir: Path) -> dict[str, Any]: + frontmatter, _ = parse_frontmatter(skill_dir / "SKILL.md") + manifest = load_json(skill_dir / "manifest.json") + interface_doc = load_yaml(skill_dir / "agents" / "interface.yaml") + name = str(frontmatter.get("name") or manifest.get("name") or skill_dir.name) + ir = find_skill_ir(skill_dir, name) + return { + "frontmatter": frontmatter, + "manifest": manifest, + "interface": interface_doc.get("interface", {}), + "compatibility": interface_doc.get("compatibility", {}), + "ir": ir, + "name": name, + "description": str(frontmatter.get("description", "")), + } + + +def target_alias(target: str) -> str: + if target in {"agent-skills", "agentskills"}: + return "agent-skills-compatible" + if target == "vscode": + return "agent-skills-compatible" + return target + + +def check_target(skill_dir: Path, target: str, evidence: dict[str, Any]) -> dict[str, Any]: + checks: list[str] = [] + failures: list[str] = [] + warnings: list[str] = [] + frontmatter = evidence["frontmatter"] + manifest = evidence["manifest"] + interface = evidence["interface"] + compatibility = evidence["compatibility"] + ir = evidence["ir"] + name = evidence["name"] + description = evidence["description"] + + add_check(checks, failures, (skill_dir / "SKILL.md").exists(), "SKILL.md exists", "Missing SKILL.md") + add_check(checks, failures, bool(frontmatter.get("name")), "frontmatter name exists", "Missing frontmatter name") + add_check(checks, failures, bool(description), "frontmatter description exists", "Missing frontmatter description") + add_check(checks, failures, len(description) <= 1024, "description length <= 1024", "description exceeds 1024 characters") + add_check(checks, failures, re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name) is not None, "name is runtime-safe", "name contains unsafe characters") + + if target in {"agent-skills", "vscode"}: + add_check(checks, failures, skill_dir.name == name, "directory name matches skill name", "directory name must match skill name") + + for field in ("name", "version", "owner", "status", "maturity_tier", "review_cadence"): + add_check(checks, failures, bool(manifest.get(field)), f"manifest.{field} exists", f"Missing manifest.{field}") + targets = [str(item) for item in manifest.get("target_platforms", [])] + alias = target_alias(target) + target_declared = alias in targets or target in targets + target_message = f"manifest declares {target}" if target in targets else f"manifest declares {target} via {alias}" + add_check(checks, failures, target_declared, target_message, f"manifest target missing: {target}") + + for field in ("display_name", "short_description", "default_prompt"): + add_check(checks, failures, bool(interface.get(field)), f"interface.{field} exists", f"Missing interface.{field}") + adapter_targets = [str(item) for item in compatibility.get("adapter_targets", [])] + if target in {"openai", "claude", "generic"}: + add_check(checks, failures, target in adapter_targets, f"adapter target declares {target}", f"adapter target missing: {target}") + else: + add_check(checks, failures, compatibility.get("canonical_format") == "agent-skills", "canonical format is agent-skills", "canonical format must be agent-skills") + + activation = compatibility.get("activation", {}) + execution = compatibility.get("execution", {}) + trust = compatibility.get("trust", {}) + degradation = compatibility.get("degradation", {}) + add_check(checks, failures, bool(activation.get("mode")), "activation mode exists", "Missing activation mode") + add_check(checks, failures, bool(execution.get("context")), "execution context exists", "Missing execution context") + add_check(checks, failures, bool(execution.get("shell")), "execution shell exists", "Missing execution shell") + add_check(checks, failures, bool(trust.get("source_tier")), "trust source tier exists", "Missing trust source tier") + if target in {"openai", "claude", "generic"}: + add_check(checks, failures, bool(degradation.get(target)), f"{target} degradation note exists", f"Missing {target} degradation note") + else: + warnings.append(f"{target} uses canonical Agent Skills metadata; runtime-specific behavior transforms are not implemented in v0.") + + add_check(checks, failures, ir.get("schema_version") == "2.0.0", "Skill IR schema_version is 2.0.0", "Missing or invalid Skill IR") + add_check(checks, failures, ir.get("name") == name, "Skill IR name matches frontmatter", "Skill IR name does not match frontmatter") + add_check( + checks, + failures, + ir.get("trigger_surface", {}).get("description") == description, + "Skill IR description matches frontmatter", + "Skill IR description does not match frontmatter", + ) + resource_checks, resource_failures = relative_resource_check(skill_dir, ir) + checks.extend(resource_checks[:12]) + failures.extend(resource_failures) + + return { + "target": target, + "status": "pass" if not failures else "fail", + "checks": checks, + "failures": failures, + "warnings": warnings, + } + + +def render_markdown(payload: dict[str, Any]) -> str: + lines = [ + "# Runtime Conformance Matrix", + "", + f"- Skill: `{payload['skill']}`", + f"- Targets: `{payload['summary']['target_count']}`", + f"- Passed: `{payload['summary']['pass_count']}`", + f"- Failed: `{payload['summary']['fail_count']}`", + "", + "| Target | Status | Failures | Warnings |", + "| --- | --- | --- | --- |", + ] + for target in payload["targets"]: + failures = "
    ".join(target["failures"]) if target["failures"] else "None" + warnings = "
    ".join(target["warnings"]) if target["warnings"] else "None" + lines.append(f"| {target['target']} | {target['status']} | {failures} | {warnings} |") + lines.extend(["", "## Reviewer Notes", "", "- Failed targets block release for that target.", "- Warnings identify lossy or not-yet-compiled behavior that must remain visible."]) + return "\n".join(lines).strip() + "\n" + + +def run_conformance(skill_dir: Path, targets: list[str], output_json: Path, output_md: Path) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + evidence = common_evidence(skill_dir) + target_results = [check_target(skill_dir, target, evidence) for target in targets] + summary = { + "target_count": len(target_results), + "pass_count": sum(1 for item in target_results if item["status"] == "pass"), + "fail_count": sum(1 for item in target_results if item["status"] == "fail"), + } + payload = { + "ok": summary["fail_count"] == 0, + "skill": evidence["name"], + "targets": target_results, + "summary": summary, + "artifacts": {"json": display_path(output_json), "markdown": display_path(output_md)}, + } + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run runtime conformance checks for Skill OS 2.0 targets.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--target", action="append", choices=DEFAULT_TARGETS) + parser.add_argument("--output-json") + parser.add_argument("--output-md") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + targets = args.target or DEFAULT_TARGETS + payload = run_conformance( + skill_dir, + targets, + Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "conformance_matrix.json", + Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "conformance_matrix.md", + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_output_eval.py b/scripts/run_output_eval.py new file mode 100644 index 00000000..2106931e --- /dev/null +++ b/scripts/run_output_eval.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_CASES = ROOT / "evals" / "output" / "cases.jsonl" + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_cases(path: Path) -> list[dict[str, Any]]: + cases = [] + for line_number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): + stripped = line.strip() + if not stripped: + continue + try: + payload = json.loads(stripped) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid JSONL at {path}:{line_number}: {exc}") from exc + if not isinstance(payload, dict): + raise ValueError(f"Output eval case at {path}:{line_number} must be an object") + cases.append(payload) + return cases + + +def normalize(text: str) -> str: + return str(text).casefold() + + +def validate_case(case: dict[str, Any]) -> list[str]: + failures = [] + for key in ("id", "prompt", "baseline_output", "with_skill_output", "assertions"): + if key not in case: + failures.append(f"{case.get('id', '')}: missing {key}") + assertions = case.get("assertions", []) + if not isinstance(assertions, list) or not assertions: + failures.append(f"{case.get('id', '')}: assertions must be a non-empty list") + for assertion in assertions if isinstance(assertions, list) else []: + if not isinstance(assertion, dict): + failures.append(f"{case.get('id', '')}: assertion must be an object") + continue + if not assertion.get("id") or not assertion.get("description"): + failures.append(f"{case.get('id', '')}: assertion id and description are required") + return failures + + +def check_assertion(output: str, assertion: dict[str, Any]) -> dict[str, Any]: + lowered = normalize(output) + required = [str(item) for item in assertion.get("required", [])] + forbidden = [str(item) for item in assertion.get("forbidden", [])] + missing = [item for item in required if normalize(item) not in lowered] + present_forbidden = [item for item in forbidden if normalize(item) in lowered] + passed = not missing and not present_forbidden + return { + "id": assertion.get("id", "assertion"), + "description": assertion.get("description", ""), + "weight": float(assertion.get("weight", 1) or 0), + "failure_type": assertion.get("failure_type", "assertion_failed"), + "passed": passed, + "missing": missing, + "present_forbidden": present_forbidden, + } + + +def grade_output(output: str, assertions: list[dict[str, Any]]) -> dict[str, Any]: + checks = [check_assertion(output, assertion) for assertion in assertions] + total_weight = sum(item["weight"] for item in checks) or len(checks) or 1 + passed_weight = sum(item["weight"] for item in checks if item["passed"]) + failed = [item for item in checks if not item["passed"]] + return { + "score": round(passed_weight / total_weight * 100, 2), + "passed_count": len(checks) - len(failed), + "failed_count": len(failed), + "checks": checks, + "failed": failed, + } + + +def grade_case(case: dict[str, Any]) -> dict[str, Any]: + assertions = case.get("assertions", []) + baseline = grade_output(str(case.get("baseline_output", "")), assertions) + with_skill = grade_output(str(case.get("with_skill_output", "")), assertions) + return { + "id": case["id"], + "prompt": case["prompt"], + "baseline": baseline, + "with_skill": with_skill, + "delta": round(with_skill["score"] - baseline["score"], 2), + "winner": "with_skill" if with_skill["score"] >= baseline["score"] else "baseline", + "failure_taxonomy": sorted({item["failure_type"] for item in with_skill["failed"]}), + } + + +def build_summary(results: list[dict[str, Any]]) -> dict[str, Any]: + case_count = len(results) + baseline_average = sum(item["baseline"]["score"] for item in results) / case_count if case_count else 0 + with_skill_average = sum(item["with_skill"]["score"] for item in results) / case_count if case_count else 0 + regressions = [item for item in results if item["delta"] < 0] + failures = sorted({failure for item in results for failure in item["failure_taxonomy"]}) + return { + "case_count": case_count, + "baseline_pass_rate": round(baseline_average, 2), + "with_skill_pass_rate": round(with_skill_average, 2), + "delta": round(with_skill_average - baseline_average, 2), + "regression_count": len(regressions), + "gate_pass": with_skill_average >= baseline_average and not regressions, + "failure_taxonomy": failures, + } + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + "# Output Quality Scorecard", + "", + "This v0 scorecard compares static without-skill and with-skill outputs using assertion grading.", + "", + f"- Cases: `{summary['case_count']}`", + f"- Baseline pass rate: `{summary['baseline_pass_rate']}`", + f"- With-skill pass rate: `{summary['with_skill_pass_rate']}`", + f"- Delta: `{summary['delta']}`", + f"- Regressions: `{summary['regression_count']}`", + f"- Gate pass: `{summary['gate_pass']}`", + "", + "## Case Results", + "", + "| Case | Baseline | With Skill | Delta | Winner | Failed With-Skill Assertions |", + "| --- | ---: | ---: | ---: | --- | --- |", + ] + for item in payload["results"]: + failed = ", ".join(failure["id"] for failure in item["with_skill"]["failed"]) or "None" + lines.append( + f"| {item['id']} | {item['baseline']['score']} | {item['with_skill']['score']} | {item['delta']} | {item['winner']} | {failed} |" + ) + lines.extend(["", "## Failure Taxonomy", ""]) + if summary["failure_taxonomy"]: + for failure in summary["failure_taxonomy"]: + lines.append(f"- {failure}") + else: + lines.append("- No with-skill assertion failures.") + lines.extend( + [ + "", + "## Next Fixes", + "", + "- Add holdout cases before using this as a release gate.", + "- Promote repeated failed assertions into the output-risk profile.", + "- Keep assertions tied to material deliverables, not phrasing trivia.", + ] + ) + return "\n".join(lines).strip() + "\n" + + +def run_output_eval(cases_path: Path, output_json: Path, output_md: Path) -> dict[str, Any]: + cases = load_cases(cases_path) + validation_failures = [failure for case in cases for failure in validate_case(case)] + if validation_failures: + payload = { + "ok": False, + "cases": display_path(cases_path), + "summary": { + "case_count": len(cases), + "baseline_pass_rate": 0, + "with_skill_pass_rate": 0, + "delta": 0, + "regression_count": 0, + "gate_pass": False, + "failure_taxonomy": ["invalid_case"], + }, + "results": [], + "failures": validation_failures, + } + else: + results = [grade_case(case) for case in cases] + payload = { + "ok": True, + "cases": display_path(cases_path), + "summary": build_summary(results), + "results": results, + "failures": [], + } + payload["artifacts"] = {"json": display_path(output_json), "markdown": display_path(output_md)} + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run Output Eval Lab assertion grading for with-skill vs baseline outputs.") + parser.add_argument("--cases", default=str(DEFAULT_CASES)) + parser.add_argument("--output-json", default=str(ROOT / "reports" / "output_quality_scorecard.json")) + parser.add_argument("--output-md", default=str(ROOT / "reports" / "output_quality_scorecard.md")) + args = parser.parse_args() + + payload = run_output_eval( + Path(args.cases).resolve(), + Path(args.output_json).resolve(), + Path(args.output_md).resolve(), + ) + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/skill_report_metrics.py b/scripts/skill_report_metrics.py index 133e9d3a..2a894a24 100644 --- a/scripts/skill_report_metrics.py +++ b/scripts/skill_report_metrics.py @@ -3,9 +3,14 @@ from pathlib import Path REPORT_EVIDENCE = [ + "skill-ir.json", "intent-dialogue.json", "intent-confidence.json", "reference-synthesis.json", + "output_quality_scorecard.json", + "conformance_matrix.json", + "security_trust_report.json", + "skill_atlas.json", "artifact-design-profile.json", "prompt-quality-profile.json", "system-model.json", @@ -117,7 +122,14 @@ def trigger_metric(skill_dir: Path) -> dict: def evidence_metric(skill_dir: Path) -> dict: reports_dir = skill_dir / "reports" - present = [name for name in REPORT_EVIDENCE if (reports_dir / name).exists()] + present = [] + for name in REPORT_EVIDENCE: + if name == "skill-ir.json": + if (reports_dir / name).exists() or any((skill_dir / "skill-ir" / "examples").glob("*.json")): + present.append(name) + continue + if (reports_dir / name).exists(): + present.append(name) score = round(len(present) / len(REPORT_EVIDENCE) * 100) reasons = [] if present: diff --git a/scripts/skill_report_model.py b/scripts/skill_report_model.py index 231287f6..f27f0e28 100644 --- a/scripts/skill_report_model.py +++ b/scripts/skill_report_model.py @@ -23,6 +23,8 @@ KNOWN_ENTRIES = [ ("reports", "Generated evidence and overview artifacts"), ] +IGNORED_PACKAGE_PARTS = {".git", "__pycache__", ".venv", "venv", "node_modules", "dist"} + REPORT_NAV_V2 = [ {"label": "技能概述", "label_en": "Overview", "href": "overview"}, {"label": "总览指标", "label_en": "Metrics", "href": "metrics"}, @@ -144,7 +146,19 @@ def package_entries(skill_dir: Path) -> list[dict]: target = skill_dir / rel_path if target.exists(): kind = "folder" if target.is_dir() else "file" - count = len([path for path in target.rglob("*") if path.is_file()]) if target.is_dir() else 1 + if target.is_dir(): + count = len( + [ + path + for path in target.rglob("*") + if path.is_file() + and not path.is_symlink() + and not any(part in IGNORED_PACKAGE_PARTS for part in path.relative_to(target).parts) + and path.suffix not in {".pyc", ".pyo"} + ] + ) + else: + count = 1 items.append({"path": rel_path, "label": label, "kind": kind, "file_count": count}) return items @@ -178,6 +192,16 @@ def compact_unique(items: list[str], limit: int = 6) -> list[str]: def derive_strengths(skill_dir: Path, metadata: dict) -> list[str]: strengths = ["触发面保持精简，并锚定在 frontmatter description。"] + if (skill_dir / "reports" / "skill-ir.json").exists() or (skill_dir / "skill-ir" / "examples").exists(): + strengths.append("已生成 Skill IR，核心语义可先于平台打包被审查和迁移。") + if (skill_dir / "reports" / "output_quality_scorecard.json").exists(): + strengths.append("已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。") + if (skill_dir / "reports" / "conformance_matrix.json").exists(): + strengths.append("已生成 Runtime Conformance Matrix，可审查目标端消费能力。") + if (skill_dir / "reports" / "security_trust_report.json").exists(): + strengths.append("已生成 Security Trust Report，可审查脚本、依赖、secret 和包完整性风险。") + if (skill_dir / "reports" / "skill_atlas.json").exists(): + strengths.append("已生成 Skill Atlas，可审查多 Skill 组合中的路由冲突、过期资产和 owner 缺口。") if (skill_dir / "agents" / "interface.yaml").exists(): strengths.append("已打包 agents/interface.yaml，便于后续做跨平台适配。") if (skill_dir / "references").exists() and any((skill_dir / "references").iterdir()): @@ -188,7 +212,7 @@ def derive_strengths(skill_dir: Path, metadata: dict) -> list[str]: strengths.append("包内包含可随 Skill 迁移的质量门禁或触发检查。") if metadata.get("maturity_tier"): strengths.append(f"生命周期元数据清晰，成熟度层级为 `{metadata['maturity_tier']}`。") - return strengths[:5] + return strengths[:6] def scenario_items(description: str, usage_steps: list[str], metadata: dict) -> list[str]: @@ -427,6 +451,14 @@ def build_report_model(skill_dir: Path) -> dict: prompt_quality = load_json(skill_dir / "reports" / "prompt-quality-profile.json") system_model = load_json(skill_dir / "reports" / "system-model.json") output_risk = load_json(skill_dir / "reports" / "output-risk-profile.json") + output_quality = load_json(skill_dir / "reports" / "output_quality_scorecard.json") + conformance = load_json(skill_dir / "reports" / "conformance_matrix.json") + trust_report = load_json(skill_dir / "reports" / "security_trust_report.json") + skill_atlas = load_json(skill_dir / "reports" / "skill_atlas.json") + skill_ir = load_json(skill_dir / "reports" / "skill-ir.json") + if not skill_ir: + example_ir = skill_dir / "skill-ir" / "examples" / f"{frontmatter.get('name', skill_dir.name)}.json" + skill_ir = load_json(example_ir) reference_synthesis = load_json(skill_dir / "reports" / "reference-synthesis.json") iteration = load_json(skill_dir / "reports" / "iteration-directions.json") @@ -460,7 +492,16 @@ def build_report_model(skill_dir: Path) -> dict: "updated_at": metadata["updated_at"], "core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。", "audience": "Skill 作者、复用团队和后续 reviewer。", - "deliverables": ["SKILL.md", "agents/interface.yaml", "reports/skill-overview.html"], + "deliverables": [ + "SKILL.md", + "agents/interface.yaml", + "reports/skill-ir.json", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.md", + "reports/security_trust_report.md", + "reports/skill_atlas.html", + "reports/skill-overview.html", + ], "flow": ["输入材料", "Skill 包体", "可复用能力"], } contract = { @@ -523,6 +564,16 @@ def build_report_model(skill_dir: Path) -> dict: "如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。", ], "benchmark_highlights": [], + "skill_ir": { + "schema_version": skill_ir.get("schema_version", ""), + "target_count": len(skill_ir.get("targets", [])), + "trigger_samples": len(skill_ir.get("trigger_surface", {}).get("should_trigger", [])), + "output_eval_cases": len(skill_ir.get("eval_plan", {}).get("output", [])), + }, + "output_quality": output_quality.get("summary", {}), + "runtime_conformance": conformance.get("summary", {}), + "trust_security": trust_report.get("summary", {}), + "skill_atlas": skill_atlas.get("summary", {}), "synthesis_highlights": synthesis, "artifact_design": q_review["artifact_design"], "prompt_quality": q_review["prompt_quality"], diff --git a/scripts/trust_check.py b/scripts/trust_check.py new file mode 100644 index 00000000..f0afeb42 --- /dev/null +++ b/scripts/trust_check.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 +import argparse +import ast +import hashlib +import json +import re +from pathlib import Path +from typing import Any + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +ROOT = Path(__file__).resolve().parent.parent +SCAN_DIRS = ["agents", "docs", "evals", "references", "runtime", "scripts", "security", "skill-ir", "templates"] +ROOT_FILES = ["SKILL.md", "README.md", "manifest.json", "requirements-ci.txt", "Makefile"] +TEXT_SUFFIXES = {".md", ".json", ".jsonl", ".yaml", ".yml", ".py", ".sh", ".txt", ".toml"} +SECRET_PATTERNS = [ + ("private_key", re.compile(r"-----BEGIN (?:RSA |DSA |EC |OPENSSH |PGP )?PRIVATE KEY-----")), + ("github_token", re.compile(r"\bgh[pousr]_[A-Za-z0-9_]{20,}\b")), + ("slack_token", re.compile(r"\bxox[baprs]-[A-Za-z0-9-]{20,}\b")), + ("aws_access_key", re.compile(r"\bAKIA[0-9A-Z]{16}\b")), + ("openai_api_key", re.compile(r"\bsk-[A-Za-z0-9]{32,}\b")), +] + + +def display_path(path: Path) -> str: + try: + return str(path.resolve().relative_to(ROOT.resolve())) + except ValueError: + return str(path.resolve()) + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_yaml(path: Path) -> dict[str, Any]: + if not path.exists() or yaml is None: + return {} + payload = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + return payload if isinstance(payload, dict) else {} + + +def iter_scan_files(skill_dir: Path) -> list[Path]: + files = [] + for rel in ROOT_FILES: + path = skill_dir / rel + if path.exists() and path.is_file(): + files.append(path) + for rel in SCAN_DIRS: + folder = skill_dir / rel + if not folder.exists(): + continue + for path in sorted(folder.rglob("*")): + if not path.is_file() or path.is_symlink(): + continue + if path.suffix in TEXT_SUFFIXES and path.stat().st_size <= 1_000_000: + files.append(path) + return sorted(set(files)) + + +def relpath(skill_dir: Path, path: Path) -> str: + return str(path.relative_to(skill_dir)) + + +def scan_secrets(skill_dir: Path, files: list[Path]) -> list[dict[str, Any]]: + findings = [] + for path in files: + text = path.read_text(encoding="utf-8", errors="replace") + for name, pattern in SECRET_PATTERNS: + for match in pattern.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + findings.append({"type": name, "path": relpath(skill_dir, path), "line": line}) + return findings + + +def script_inventory(skill_dir: Path) -> list[dict[str, Any]]: + scripts_dir = skill_dir / "scripts" + if not scripts_dir.exists(): + return [] + inventory = [] + for path in sorted(scripts_dir.glob("*.py")): + text = path.read_text(encoding="utf-8", errors="replace") + flags = script_flags(text) + inventory.append( + { + "path": relpath(skill_dir, path), + "has_argparse": "argparse" in text, + "has_main_guard": 'if __name__ == "__main__"' in text, + "uses_input": flags["uses_input"], + "uses_network": flags["uses_network"], + "uses_subprocess": flags["uses_subprocess"], + } + ) + return inventory + + +def call_name(node: ast.AST) -> str: + if isinstance(node, ast.Name): + return node.id + if isinstance(node, ast.Attribute): + base = call_name(node.value) + return f"{base}.{node.attr}" if base else node.attr + return "" + + +def script_flags(text: str) -> dict[str, bool]: + try: + tree = ast.parse(text) + except SyntaxError: + return { + "uses_input": bool(re.search(r"\b(?:input|getpass)\s*\(", text)), + "uses_network": bool(re.search(r"\b(?:urlopen|Request)\s*\(|\brequests\.", text)), + "uses_subprocess": "subprocess." in text, + } + calls = [call_name(node.func) for node in ast.walk(tree) if isinstance(node, ast.Call)] + return { + "uses_input": any(name in {"input", "getpass.getpass"} or name.endswith(".getpass") for name in calls), + "uses_network": any(name in {"urlopen", "Request"} or name.startswith("requests.") for name in calls), + "uses_subprocess": any(name.startswith("subprocess.") for name in calls), + } + + +def dependency_status(skill_dir: Path) -> dict[str, Any]: + candidates = ["requirements-ci.txt", "requirements.txt", "pyproject.toml", "package-lock.json", "uv.lock", "poetry.lock"] + present = [name for name in candidates if (skill_dir / name).exists()] + pinned = [] + unpinned = [] + requirements = skill_dir / "requirements-ci.txt" + if requirements.exists(): + for line in requirements.read_text(encoding="utf-8").splitlines(): + stripped = line.strip() + if not stripped or stripped.startswith("#"): + continue + if "==" in stripped or " @ " in stripped: + pinned.append(stripped) + else: + unpinned.append(stripped) + return {"present": present, "pinned": pinned, "unpinned": unpinned} + + +def interface_trust(skill_dir: Path) -> dict[str, Any]: + interface = load_yaml(skill_dir / "agents" / "interface.yaml") + trust = interface.get("compatibility", {}).get("trust", {}) + return { + "source_tier": trust.get("source_tier", ""), + "remote_inline_execution": trust.get("remote_inline_execution", ""), + "remote_metadata_policy": trust.get("remote_metadata_policy", ""), + } + + +def package_digest(files: list[Path], skill_dir: Path) -> str: + digest = hashlib.sha256() + for path in files: + rel = relpath(skill_dir, path) + digest.update(rel.encode("utf-8")) + digest.update(b"\0") + digest.update(hashlib.sha256(path.read_bytes()).hexdigest().encode("ascii")) + digest.update(b"\0") + return digest.hexdigest() + + +def build_trust_report(skill_dir: Path) -> dict[str, Any]: + skill_dir = skill_dir.resolve() + files = iter_scan_files(skill_dir) + secrets = scan_secrets(skill_dir, files) + scripts = script_inventory(skill_dir) + deps = dependency_status(skill_dir) + trust = interface_trust(skill_dir) + failures = [] + warnings = [] + + if secrets: + failures.append(f"High-risk secret patterns found: {len(secrets)}") + if trust.get("remote_inline_execution") not in {"forbid", "deny", "false"}: + failures.append("remote_inline_execution must be forbid for governed release") + if deps["unpinned"]: + warnings.append(f"Unpinned dependency entries: {', '.join(deps['unpinned'])}") + if not deps["present"]: + warnings.append("No dependency or lock file detected") + missing_help = [item["path"] for item in scripts if not item["has_argparse"]] + if missing_help: + warnings.append(f"Scripts without argparse/help surface: {', '.join(missing_help[:8])}") + interactive = [item["path"] for item in scripts if item["uses_input"]] + if interactive: + warnings.append(f"Interactive scripts require reviewer awareness: {', '.join(interactive[:8])}") + network = [item["path"] for item in scripts if item["uses_network"]] + if network: + warnings.append(f"Network-capable scripts require bounded host policy: {', '.join(network[:8])}") + + summary = { + "scanned_files": len(files), + "script_count": len(scripts), + "secret_findings": len(secrets), + "dependency_files": deps["present"], + "network_script_count": len(network), + "interactive_script_count": len(interactive), + "package_sha256": package_digest(files, skill_dir), + } + return { + "ok": not failures, + "skill_dir": display_path(skill_dir), + "summary": summary, + "failures": failures, + "warnings": warnings, + "secrets": secrets, + "scripts": scripts, + "dependencies": deps, + "trust_metadata": trust, + } + + +def render_markdown(payload: dict[str, Any]) -> str: + summary = payload["summary"] + lines = [ + "# Security Trust Report", + "", + f"- OK: `{payload['ok']}`", + f"- Scanned files: `{summary['scanned_files']}`", + f"- Scripts: `{summary['script_count']}`", + f"- Secret findings: `{summary['secret_findings']}`", + f"- Network-capable scripts: `{summary['network_script_count']}`", + f"- Interactive scripts: `{summary['interactive_script_count']}`", + f"- Package SHA256: `{summary['package_sha256']}`", + "", + "## Failures", + "", + ] + lines.extend([f"- {item}" for item in payload["failures"]] or ["- None"]) + lines.extend(["", "## Warnings", ""]) + lines.extend([f"- {item}" for item in payload["warnings"]] or ["- None"]) + lines.extend(["", "## Dependency Evidence", ""]) + lines.append(f"- Files: `{', '.join(payload['dependencies']['present']) or 'none'}`") + lines.append(f"- Pinned entries: `{len(payload['dependencies']['pinned'])}`") + lines.append(f"- Unpinned entries: `{len(payload['dependencies']['unpinned'])}`") + lines.extend(["", "## Script Surface", "", "| Script | Argparse | Main Guard | Input | Network | Subprocess |", "| --- | --- | --- | --- | --- | --- |"]) + for item in payload["scripts"]: + lines.append( + f"| {item['path']} | {item['has_argparse']} | {item['has_main_guard']} | {item['uses_input']} | {item['uses_network']} | {item['uses_subprocess']} |" + ) + return "\n".join(lines).strip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run Skill OS trust and security checks.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json") + parser.add_argument("--output-md") + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + output_json = Path(args.output_json).resolve() if args.output_json else skill_dir / "reports" / "security_trust_report.json" + output_md = Path(args.output_md).resolve() if args.output_md else skill_dir / "reports" / "security_trust_report.md" + payload = build_trust_report(skill_dir) + payload["artifacts"] = {"json": display_path(output_json), "markdown": display_path(output_md)} + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(payload), encoding="utf-8") + print(json.dumps(payload, ensure_ascii=False, indent=2)) + raise SystemExit(0 if payload["ok"] else 2) + + +if __name__ == "__main__": + main() diff --git a/scripts/yao.py b/scripts/yao.py index 2658e694..1f569580 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -858,6 +858,71 @@ def command_baseline_compare(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_skill_ir(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.validate_only: + cmd.append("--validate-only") + result = run_script("export_skill_ir.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_output_eval(args: argparse.Namespace) -> int: + cmd = [] + if args.cases: + cmd.extend(["--cases", args.cases]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + result = run_script("run_output_eval.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_conformance(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + for target in args.target or []: + cmd.extend(["--target", target]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + result = run_script("run_conformance_suite.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_trust(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + result = run_script("trust_check.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_skill_atlas(args: argparse.Namespace) -> int: + cmd = ["--workspace-root", str(Path(args.workspace_root).resolve())] + if args.output_dir: + cmd.extend(["--output-dir", args.output_dir]) + if args.report_html: + cmd.extend(["--report-html", args.report_html]) + if args.report_json: + cmd.extend(["--report-json", args.report_json]) + if args.overlap_threshold is not None: + cmd.extend(["--overlap-threshold", str(args.overlap_threshold)]) + if args.today: + cmd.extend(["--today", args.today]) + result = run_script("build_skill_atlas.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_review(args: argparse.Namespace) -> int: target_name = resolve_promotion_target(args.target) bundle_dir = ROOT / "reports" / "iteration_bundles" / target_name @@ -1246,6 +1311,40 @@ def build_parser() -> argparse.ArgumentParser: ) baseline_compare_cmd.set_defaults(func=command_baseline_compare) + skill_ir_cmd = subparsers.add_parser("skill-ir", help="Export platform-neutral Skill IR for a skill package.") + skill_ir_cmd.add_argument("skill_dir", nargs="?", default=".") + skill_ir_cmd.add_argument("--output-json") + skill_ir_cmd.add_argument("--validate-only", action="store_true") + skill_ir_cmd.set_defaults(func=command_skill_ir) + + output_eval_cmd = subparsers.add_parser("output-eval", help="Run Output Eval Lab assertion grading.") + output_eval_cmd.add_argument("--cases") + output_eval_cmd.add_argument("--output-json") + output_eval_cmd.add_argument("--output-md") + output_eval_cmd.set_defaults(func=command_output_eval) + + conformance_cmd = subparsers.add_parser("conformance", help="Run runtime conformance checks for Skill OS targets.") + conformance_cmd.add_argument("skill_dir", nargs="?", default=".") + conformance_cmd.add_argument("--target", action="append", choices=["openai", "claude", "agent-skills", "vscode", "generic"]) + conformance_cmd.add_argument("--output-json") + conformance_cmd.add_argument("--output-md") + conformance_cmd.set_defaults(func=command_conformance) + + trust_cmd = subparsers.add_parser("trust", help="Run trust and security checks for a skill package.") + trust_cmd.add_argument("skill_dir", nargs="?", default=".") + trust_cmd.add_argument("--output-json") + trust_cmd.add_argument("--output-md") + trust_cmd.set_defaults(func=command_trust) + + skill_atlas_cmd = subparsers.add_parser("skill-atlas", help="Build a portfolio-level Skill Atlas for a workspace.") + skill_atlas_cmd.add_argument("--workspace-root", default=".") + skill_atlas_cmd.add_argument("--output-dir") + skill_atlas_cmd.add_argument("--report-html") + skill_atlas_cmd.add_argument("--report-json") + skill_atlas_cmd.add_argument("--overlap-threshold", type=float) + skill_atlas_cmd.add_argument("--today") + skill_atlas_cmd.set_defaults(func=command_skill_atlas) + package_cmd = subparsers.add_parser("package", help="Export compatibility artifacts for selected targets.") package_cmd.add_argument("skill_dir", nargs="?", default=".") package_cmd.add_argument("--platform", action="append") diff --git a/security/dependency_policy.md b/security/dependency_policy.md new file mode 100644 index 00000000..819c2db3 --- /dev/null +++ b/security/dependency_policy.md @@ -0,0 +1,10 @@ +# Dependency Policy + +Runtime dependencies should be pinned for CI and release gates. + +Current v0 policy: + +- Python CI dependencies belong in `requirements-ci.txt`. +- Release-critical dependencies should use exact pins. +- Generated packages should avoid fetching code during normal execution. +- Any future package manager lockfile should be included in the trust report. diff --git a/security/script_policy.md b/security/script_policy.md new file mode 100644 index 00000000..f1fd4d41 --- /dev/null +++ b/security/script_policy.md @@ -0,0 +1,11 @@ +# Script Policy + +Scripts should be deterministic, non-destructive by default, and reviewable. + +Minimum expectations: + +- expose a command-line entrypoint or clear function boundary +- prefer `argparse` and `--help` for operator-facing scripts +- avoid hidden interactive prompts unless the command is explicitly interactive +- state network or filesystem side effects in nearby docs or trust metadata +- emit structured JSON when used by `scripts/yao.py` diff --git a/security/secret_policy.md b/security/secret_policy.md new file mode 100644 index 00000000..1ddf6342 --- /dev/null +++ b/security/secret_policy.md @@ -0,0 +1,7 @@ +# Secret Policy + +Skills must not commit credentials, tokens, private keys, or customer secrets. + +The v0 scanner blocks obvious high-risk patterns such as private key blocks, GitHub personal access tokens, Slack bot tokens, AWS access key ids, and OpenAI-style API keys. + +Potential private paths or local-only references should be treated as warnings unless they expose credentials. diff --git a/security/trust_policy.md b/security/trust_policy.md new file mode 100644 index 00000000..c1e0205a --- /dev/null +++ b/security/trust_policy.md @@ -0,0 +1,14 @@ +# Trust Policy + +Yao Meta Skill treats scripts and distributed skill packages as supply-chain surfaces. + +Governed or team-distributed skills should produce a trust report before release. The report must identify: + +- secret exposure risk +- scripts that lack a visible CLI/help surface +- interactive or network-capable scripts +- dependency pinning status +- runtime trust metadata +- package integrity hash + +High-risk secrets or unrestricted remote inline execution block governed release. diff --git a/skill-ir/README.md b/skill-ir/README.md new file mode 100644 index 00000000..ad33bfad --- /dev/null +++ b/skill-ir/README.md @@ -0,0 +1,24 @@ +# Skill IR + +Skill IR is the platform-neutral capability contract for Yao Meta Skill 2.0. + +The IR keeps core skill semantics stable before any platform-specific packaging: + +- job to be done +- trigger surface and exclusions +- workflow steps and failure modes +- references, scripts, assets, and reports +- trigger and output evaluation plan +- risk and governance metadata + +Generate the root example: + +```bash +python3 scripts/export_skill_ir.py . --output-json skill-ir/examples/yao-meta-skill.json +``` + +Validate the v0 implementation: + +```bash +python3 tests/verify_skill_ir.py +``` diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json new file mode 100644 index 00000000..549bd743 --- /dev/null +++ b/skill-ir/examples/yao-meta-skill.json @@ -0,0 +1,257 @@ +{ + "schema_version": "2.0.0", + "name": "yao-meta-skill", + "title": "Yao Meta Skill", + "job_to_be_done": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "trigger_surface": { + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Turn this runbook into a reusable agent skill.", + "Convert this process note into a reusable skill package for the team.", + "Improve this skill description and add evals.", + "Refactor this prompt into a proper skill package.", + "Package this skill for team reuse.", + "Create a meta-skill for packaging internal workflows.", + "Standardize this recurring workflow as a shareable skill package with references." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Just explain what a skill is.", + "Summarize this random note.", + "Translate this README into Japanese.", + "Write a product headline for this landing page.", + "Draft a blog title for this article.", + "Rewrite this paragraph more clearly, but do not package anything.", + "Explain the difference between a runbook and a workflow.", + "one-off adjacent requests that do not match the recurring job", + "private local material that was not intentionally included" + ], + "edge_cases": [ + "Create a one-off prompt for this task.", + "Write a custom answer for this request without creating a skill package.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.", + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill." + ] + }, + "workflow": { + "steps": [ + "Decide whether the request should become a skill and choose the lightest fit.", + "Capture job, output, exclusions, constraints, and standards.", + "Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.", + "Write the `description` early and test route quality before expanding the package.", + "Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter.", + "Add only folders and gates that earn their keep.", + "Surface the top three next iteration directions." + ], + "decision_points": [ + "Route by frontmatter `description`.", + "Keep `SKILL.md` lean; put guidance in `references/`, logic in `scripts/`, and evidence in `reports/`.", + "Use the lightest reliable process.", + "`Scaffold`: exploratory or personal.", + "`Production`: team reuse.", + "`Library`: shared infrastructure.", + "a working skill directory", + "a `SKILL.md`", + "aligned `agents/interface.yaml`", + "optional `references/`, `scripts/`, `evals/`, `reports/`, and `manifest.json` when justified" + ], + "failure_modes": [ + "Tables can render as dense grids with weak hierarchy or poor mobile readability.", + "Long bullets can make the output look complete while hiding the actual decision logic.", + "Footnote markers or dense citation notes can interrupt the reading flow.", + "Evidence can be over-attached to obvious statements and under-attached to risky claims.", + "Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.", + "Missing screenshots can cause the skill to invent visual references instead of declaring the gap.", + "Users start invoking the skill for adjacent one-off or explanation-only requests.", + "Outputs remain valid but become generic, cluttered, or weakly aligned with the user's domain.", + "Borrowed benchmark patterns no longer fit the local job or add ceremony without payoff.", + "Skill usage becomes team-critical while ownership, review cadence, or rollback evidence stays informal." + ] + }, + "resources": { + "references": [ + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/trust-security-method.md" + ], + "scripts": [ + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/optimize_description.py", + "scripts/promotion_checker.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_viewer.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/validate_skill.py", + "scripts/yao.py" + ], + "assets": [ + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "reports": [ + "reports/benchmark_methodology.md", + "reports/intent-context.json", + "reports/intent-confidence.json", + "reports/intent-confidence.md", + "reports/reference-synthesis.json", + "reports/reference-synthesis.md", + "reports/output-risk-profile.json", + "reports/output-risk-profile.md", + "reports/artifact-design-profile.json", + "reports/artifact-design-profile.md", + "reports/prompt-quality-profile.json", + "reports/prompt-quality-profile.md", + "reports/system-model.json", + "reports/system-model.md", + "reports/iteration-directions.json", + "reports/iteration-directions.md", + "reports/skill-overview.json", + "reports/skill-overview.html", + "reports/output_quality_scorecard.json", + "reports/output_quality_scorecard.md", + "reports/conformance_matrix.json", + "reports/conformance_matrix.md", + "reports/security_trust_report.json", + "reports/security_trust_report.md", + "reports/skill_atlas.json", + "reports/skill_atlas.html", + "reports/skill-os-2-review.md", + "reports/portability_score.json", + "reports/portability_score.md", + "reports/governance_score.json" + ] + }, + "eval_plan": { + "trigger": [ + "evals/README.md", + "evals/adversarial/trigger_cases.json", + "evals/blind_holdout/trigger_cases.json", + "evals/confusion/route_cases.json", + "evals/dev/trigger_cases.json", + "evals/failure-cases.md", + "evals/history/2026-03-31-context-first-pack.json", + "evals/history/2026-03-31-family-suite-expansion.json", + "evals/history/2026-03-31-governance-quality-gates.json", + "evals/history/2026-03-31-governed-assets-and-history.json", + "evals/history/2026-04-01-route-scorecard-foundation.json", + "evals/history/README.md", + "evals/history/description_optimization/2026-03-31-description-optimization-suite.json", + "evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json", + "evals/history/description_optimization/2026-04-01-blind-holdout-and-drift-history.json", + "evals/history/description_optimization/2026-04-01-judge-backed-blind-eval.json", + "evals/history/description_optimization/README.md", + "evals/holdout/trigger_cases.json", + "evals/output/cases.jsonl", + "evals/output/schema.json", + "evals/packaging_expectations.json", + "evals/promotion_policy.md", + "evals/semantic_config.json", + "evals/train/trigger_cases.json", + "evals/trigger_cases.json" + ], + "output": [ + "evals/output/cases.jsonl", + "evals/output/schema.json" + ], + "adversarial": [ + "evals/adversarial/trigger_cases.json" + ], + "baseline": "without_skill" + }, + "risk": { + "output_risk": "high", + "execution_risk": "medium", + "trust_boundary": "external" + }, + "governance": { + "owner": "Yao Team", + "maturity": "governed", + "review_cadence": "quarterly", + "review_due": "" + }, + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "source_files": [ + "SKILL.md", + "manifest.json", + "agents/interface.yaml", + "reports/intent-context.json", + "reports/output-risk-profile.json", + "reports/system-model.json" + ] +} diff --git a/skill-ir/schema.json b/skill-ir/schema.json new file mode 100644 index 00000000..1011d1f5 --- /dev/null +++ b/skill-ir/schema.json @@ -0,0 +1,156 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.example/skill-ir.schema.json", + "title": "Yao Skill IR", + "description": "Platform-neutral capability contract for Yao Meta Skill 2.0.", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "name", + "job_to_be_done", + "trigger_surface", + "workflow", + "resources", + "eval_plan", + "risk", + "governance" + ], + "properties": { + "schema_version": { + "const": "2.0.0" + }, + "name": { + "type": "string", + "minLength": 1 + }, + "title": { + "type": "string" + }, + "job_to_be_done": { + "type": "string", + "minLength": 1 + }, + "trigger_surface": { + "type": "object", + "additionalProperties": false, + "required": ["description", "should_trigger", "should_not_trigger", "edge_cases"], + "properties": { + "description": {"type": "string", "minLength": 1}, + "should_trigger": { + "type": "array", + "items": {"type": "string"} + }, + "should_not_trigger": { + "type": "array", + "items": {"type": "string"} + }, + "edge_cases": { + "type": "array", + "items": {"type": "string"} + } + } + }, + "workflow": { + "type": "object", + "additionalProperties": false, + "required": ["steps", "decision_points", "failure_modes"], + "properties": { + "steps": { + "type": "array", + "items": {"type": "string"} + }, + "decision_points": { + "type": "array", + "items": {"type": "string"} + }, + "failure_modes": { + "type": "array", + "items": {"type": "string"} + } + } + }, + "resources": { + "type": "object", + "additionalProperties": false, + "required": ["references", "scripts", "assets", "reports"], + "properties": { + "references": { + "type": "array", + "items": {"type": "string"} + }, + "scripts": { + "type": "array", + "items": {"type": "string"} + }, + "assets": { + "type": "array", + "items": {"type": "string"} + }, + "reports": { + "type": "array", + "items": {"type": "string"} + } + } + }, + "eval_plan": { + "type": "object", + "additionalProperties": false, + "required": ["trigger", "output", "adversarial", "baseline"], + "properties": { + "trigger": { + "type": "array", + "items": {"type": "string"} + }, + "output": { + "type": "array", + "items": {"type": "string"} + }, + "adversarial": { + "type": "array", + "items": {"type": "string"} + }, + "baseline": { + "type": "string" + } + } + }, + "risk": { + "type": "object", + "additionalProperties": false, + "required": ["output_risk", "execution_risk", "trust_boundary"], + "properties": { + "output_risk": { + "enum": ["low", "medium", "high"] + }, + "execution_risk": { + "enum": ["low", "medium", "high"] + }, + "trust_boundary": { + "enum": ["personal", "team", "external"] + } + } + }, + "governance": { + "type": "object", + "additionalProperties": false, + "required": ["owner", "maturity", "review_cadence", "review_due"], + "properties": { + "owner": {"type": "string"}, + "maturity": { + "enum": ["scaffold", "production", "library", "governed"] + }, + "review_cadence": {"type": "string"}, + "review_due": {"type": "string"} + } + }, + "targets": { + "type": "array", + "items": {"type": "string"} + }, + "source_files": { + "type": "array", + "items": {"type": "string"} + } + } +} diff --git a/skill_atlas/README.md b/skill_atlas/README.md new file mode 100644 index 00000000..4724b026 --- /dev/null +++ b/skill_atlas/README.md @@ -0,0 +1,20 @@ +# Skill Atlas + +Skill Atlas is the Skill OS 2.0 layer for managing a workspace of multiple skills. + +It detects: + +- catalog metadata +- route overlap +- shared resource dependencies +- stale skills +- missing owner / review metadata +- no-route opportunities from failure notes + +Run against a workspace: + +```bash +python3 scripts/build_skill_atlas.py --workspace-root .. --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json +``` + +The script is local-first and does not modify scanned skills. diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json new file mode 100644 index 00000000..db86f898 --- /dev/null +++ b/skill_atlas/catalog.json @@ -0,0 +1,504 @@ +{ + "workspace_root": ".", + "generated_at": "2026-06-13", + "skills": [ + { + "name": "yao-meta-skill", + "path": ".", + "description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "owner": "Yao Team", + "version": "1.1.0", + "status": "active", + "maturity": "governed", + "updated_at": "2026-03-31", + "review_cadence": "quarterly", + "targets": [ + "openai", + "claude", + "generic", + "agent-skills-compatible" + ], + "resources": [ + "scripts/build_confusion_matrix.py", + "scripts/build_skill_atlas.py", + "scripts/check_update.py", + "scripts/ci_test.py", + "scripts/collect_feedback.py", + "scripts/context_sizer.py", + "scripts/create_iteration_snapshot.py", + "scripts/cross_packager.py", + "scripts/diff_eval.py", + "scripts/export_skill_ir.py", + "scripts/github_benchmark_scan.py", + "scripts/governance_check.py", + "scripts/init_skill.py", + "scripts/judge_blind_eval.py", + "scripts/lint_skill.py", + "scripts/optimize_description.py", + "scripts/promotion_checker.py", + "scripts/render_artifact_design_profile.py", + "scripts/render_baseline_compare.py", + "scripts/render_context_reports.py", + "scripts/render_description_drift_history.py", + "scripts/render_eval_dashboard.py", + "scripts/render_intent_confidence.py", + "scripts/render_intent_dialogue.py", + "scripts/render_iteration_directions.py", + "scripts/render_iteration_ledger.py", + "scripts/render_output_risk_profile.py", + "scripts/render_portability_report.py", + "scripts/render_prompt_quality_profile.py", + "scripts/render_reference_scan.py", + "scripts/render_reference_synthesis.py", + "scripts/render_regression_history.py", + "scripts/render_review_viewer.py", + "scripts/render_skill_overview.py", + "scripts/render_social_preview.py", + "scripts/render_system_model.py", + "scripts/resource_boundary_check.py", + "scripts/run_conformance_suite.py", + "scripts/run_description_optimization_suite.py", + "scripts/run_eval_suite.py", + "scripts/run_output_eval.py", + "scripts/skill_report_charts.py", + "scripts/skill_report_metrics.py", + "scripts/skill_report_model.py", + "scripts/sync_local_install.py", + "scripts/trigger_eval.py", + "scripts/trust_check.py", + "scripts/validate_skill.py", + "scripts/yao.py", + "references/artifact-design-doctrine.md", + "references/authoring-discipline.md", + "references/eval-playbook.md", + "references/gate-selection.md", + "references/governance.md", + "references/human-review-template.md", + "references/intent-dialogue.md", + "references/iteration-philosophy.md", + "references/non-skill-decision-tree.md", + "references/operating-modes.md", + "references/output-eval-method.md", + "references/output-quality-risk.md", + "references/output-visual-quality.md", + "references/packaging-contracts.md", + "references/pattern-extraction-doctrine.md", + "references/platform-capability-matrix.md", + "references/prompt-engineering-doctrine.md", + "references/qa-ladder.md", + "references/reference-scan.md", + "references/regression-cause-taxonomy.md", + "references/resource-boundaries.md", + "references/runtime-conformance-method.md", + "references/skill-archetypes.md", + "references/skill-atlas-method.md", + "references/skill-engineering-method.md", + "references/skill-ir-method.md", + "references/systems-thinking-doctrine.md", + "references/trust-security-method.md", + "templates/basic_skill.md.j2", + "templates/complex_skill.md.j2" + ], + "token_set": [ + "add", + "asked", + "docs", + "evals", + "evaluate", + "existing", + "improve", + "notes", + "or", + "package", + "process", + "prompts", + "refactor", + "repeated", + "reuse", + "team", + "transcripts", + "workflows" + ] + }, + { + "name": "release-orchestrator", + "path": "examples/complex-release-orchestrator/generated-skill", + "description": "Coordinate software release preparation, rollout readiness, migration notes, rollback planning, stakeholder communication, and final go/no-go release packets. Use when asked to prepare a release packet, review release readiness, or turn scattered launch notes into one reusable release workflow. Do not use for generic project planning, one-off announcement drafting, or high-level release theory.", + "owner": "Yao Team", + "version": "1.0.0", + "status": "active", + "maturity": "production", + "updated_at": "2026-03-31", + "review_cadence": "quarterly", + "targets": [], + "resources": [ + "scripts/build_release_packet.py", + "references/migration-template.md", + "references/release-checklist.md", + "references/risk-matrix.md", + "references/stakeholder-comms.md" + ], + "token_set": [ + "announcement", + "asked", + "communication", + "coordinate", + "do", + "drafting", + "final", + "generic", + "go", + "high-level", + "launch", + "migration", + "no-go", + "not", + "notes", + "one", + "one-off", + "or", + "packet", + "packets", + "planning", + "preparation", + "prepare", + "project", + "readiness", + "release", + "review", + "rollback", + "rollout", + "scattered", + "software", + "stakeholder", + "theory", + "workflow" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "a11y", + "accessibility", + "asked", + "code", + "common", + "components", + "forms", + "frontend", + "issues", + "missing", + "or", + "pre-merge", + "quality", + "react", + "regressions", + "review", + "risky", + "security", + "states", + "ui", + "ux" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "description": "Review frontend code before merge.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "before", + "code", + "frontend", + "merge", + "review" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "description": "Review frontend code for accessibility, risky UI security issues, missing loading and error states, and pre-merge regressions. Use when asked to review React, UI, forms, frontend components, or merge-readiness for the frontend.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "accessibility", + "asked", + "code", + "components", + "error", + "forms", + "frontend", + "issues", + "loading", + "merge-readiness", + "missing", + "or", + "pre-merge", + "react", + "regressions", + "review", + "risky", + "security", + "states", + "ui" + ] + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill", + "description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "owner": "Incident Operations Team", + "version": "1.0.0", + "status": "active", + "maturity": "governed", + "updated_at": "2026-03-31", + "review_cadence": "monthly", + "targets": [ + "openai", + "claude", + "generic" + ], + "resources": [ + "scripts/build_incident_packet.py", + "references/comms-policy.md", + "references/review-policy.md", + "references/severity-matrix.md" + ], + "token_set": [ + "asked", + "assemble", + "assessment", + "build", + "command", + "communication", + "governed", + "incident", + "or", + "packets", + "review", + "run", + "severity", + "standardize" + ] + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "description": "Clean messy meeting notes into structured markdown summaries. Use when asked to organize meeting notes, extract action items, separate decisions from open questions, or turn rough notes into a clean recap.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "action", + "asked", + "clean", + "decisions", + "extract", + "items", + "markdown", + "meeting", + "messy", + "notes", + "open", + "or", + "organize", + "questions", + "recap", + "rough", + "separate", + "structured", + "summaries" + ] + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [ + "references/checklist.md" + ], + "token_set": [ + "a11y", + "accessibility", + "asked", + "check", + "code", + "components", + "frontend", + "missing", + "or", + "pre-merge", + "react", + "regressions", + "rendering", + "review", + "run", + "security", + "states", + "ui", + "unsafe", + "ux" + ] + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "description": "Create GEO ranking-article packs from brand briefs. Use when asked to research a brand, find GEO or 生成式引擎优化 keywords and five competitors, generate timely titles, write five ~2000-character articles, and save them to a fixed knowledge-base folder with an HTML index. Do not use for generic SEO advice, one-off copy, or writing without competitor research.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [ + "scripts/build_html_index.py", + "references/article-blueprint.md", + "references/feishu-benchmark-patterns.md", + "references/geo-research-playbook.md", + "templates/brand-brief-template.md" + ], + "token_set": [ + "2000-character", + "advice", + "articles", + "asked", + "brand", + "briefs", + "competitor", + "competitors", + "copy", + "do", + "find", + "five", + "fixed", + "folder", + "generate", + "generic", + "geo", + "html", + "index", + "keywords", + "knowledge-base", + "not", + "one-off", + "or", + "packs", + "ranking-article", + "research", + "save", + "seo", + "them", + "timely", + "titles", + "without", + "write", + "writing", + "生成式引擎优化" + ] + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "description": "Test fixture for governance validation.", + "owner": "", + "version": "1.0.0", + "status": "frozen", + "maturity": "governed", + "updated_at": "2026/03/31", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "fixture", + "governance", + "test", + "validation" + ] + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "description": "Broken YAML fixture for packager failure testing.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "broken", + "failure", + "fixture", + "packager", + "testing", + "yaml" + ] + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "description": "Broken fixture for packager failure testing.", + "owner": "", + "version": "", + "status": "", + "maturity": "", + "updated_at": "", + "review_cadence": "", + "targets": [], + "resources": [], + "token_set": [ + "broken", + "failure", + "fixture", + "packager", + "testing" + ] + } + ], + "summary": { + "skill_count": 12, + "route_collision_count": 5, + "owner_gap_count": 9, + "stale_count": 10, + "shared_resource_count": 0, + "no_route_opportunity_count": 0 + } +} diff --git a/skill_atlas/catalog.schema.json b/skill_atlas/catalog.schema.json new file mode 100644 index 00000000..00052299 --- /dev/null +++ b/skill_atlas/catalog.schema.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://yao.example/skill-atlas-catalog.schema.json", + "title": "Yao Skill Atlas Catalog", + "type": "object", + "additionalProperties": false, + "required": ["workspace_root", "generated_at", "skills", "summary"], + "properties": { + "workspace_root": {"type": "string"}, + "generated_at": {"type": "string"}, + "skills": { + "type": "array", + "items": { + "type": "object", + "required": ["name", "path", "description"], + "additionalProperties": true, + "properties": { + "name": {"type": "string"}, + "path": {"type": "string"}, + "description": {"type": "string"}, + "owner": {"type": "string"}, + "maturity": {"type": "string"}, + "updated_at": {"type": "string"}, + "review_cadence": {"type": "string"} + } + } + }, + "summary": { + "type": "object", + "additionalProperties": true + } + } +} diff --git a/skill_atlas/dependency_graph.json b/skill_atlas/dependency_graph.json new file mode 100644 index 00000000..384e236d --- /dev/null +++ b/skill_atlas/dependency_graph.json @@ -0,0 +1,53 @@ +{ + "nodes": [ + { + "name": "yao-meta-skill", + "path": "." + }, + { + "name": "release-orchestrator", + "path": "examples/complex-release-orchestrator/generated-skill" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1" + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill" + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill" + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill" + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator" + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest" + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml" + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field" + } + ], + "shared_resources": [] +} diff --git a/skill_atlas/no_route_opportunities.json b/skill_atlas/no_route_opportunities.json new file mode 100644 index 00000000..fe51488c --- /dev/null +++ b/skill_atlas/no_route_opportunities.json @@ -0,0 +1 @@ +[] diff --git a/skill_atlas/owner_review_gaps.json b/skill_atlas/owner_review_gaps.json new file mode 100644 index 00000000..ec951f44 --- /dev/null +++ b/skill_atlas/owner_review_gaps.json @@ -0,0 +1,82 @@ +[ + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "missing": [ + "owner", + "review_cadence" + ] + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "missing": [ + "owner", + "review_cadence", + "maturity" + ] + } +] diff --git a/skill_atlas/route_overlap_matrix.csv b/skill_atlas/route_overlap_matrix.csv new file mode 100644 index 00000000..33d5a89a --- /dev/null +++ b/skill_atlas/route_overlap_matrix.csv @@ -0,0 +1,67 @@ +skill_a,skill_b,path_a,path_b,score,status +yao-meta-skill,release-orchestrator,.,examples/complex-release-orchestrator/generated-skill,0.061,clear +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/final,0.054,clear +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v0,0.0,clear +yao-meta-skill,frontend-review,.,examples/evolution-frontend-review/v1,0.056,clear +yao-meta-skill,incident-command-governor,.,examples/governed-incident-command/generated-skill,0.067,clear +yao-meta-skill,note-cleanup,.,examples/simple-note-cleanup/generated-skill,0.088,clear +yao-meta-skill,frontend-review,.,examples/team-frontend-review/generated-skill,0.056,clear +yao-meta-skill,geo-ranking-article-generator,.,geo-ranking-article-generator,0.038,clear +yao-meta-skill,invalid-governance-skill,.,tests/fixtures/governance_invalid_manifest,0.0,clear +yao-meta-skill,broken-yaml-skill,.,tests/fixtures/package_invalid_yaml,0.0,clear +yao-meta-skill,broken-skill,.,tests/fixtures/package_missing_interface_field,0.0,clear +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/final,0.058,clear +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/v0,0.026,clear +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/evolution-frontend-review/v1,0.059,clear +release-orchestrator,incident-command-governor,examples/complex-release-orchestrator/generated-skill,examples/governed-incident-command/generated-skill,0.116,clear +release-orchestrator,note-cleanup,examples/complex-release-orchestrator/generated-skill,examples/simple-note-cleanup/generated-skill,0.06,clear +release-orchestrator,frontend-review,examples/complex-release-orchestrator/generated-skill,examples/team-frontend-review/generated-skill,0.059,clear +release-orchestrator,geo-ranking-article-generator,examples/complex-release-orchestrator/generated-skill,geo-ranking-article-generator,0.094,clear +release-orchestrator,invalid-governance-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear +release-orchestrator,broken-yaml-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear +release-orchestrator,broken-skill,examples/complex-release-orchestrator/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear +frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/evolution-frontend-review/v0,0.13,clear +frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/evolution-frontend-review/v1,0.708,collision +frontend-review,incident-command-governor,examples/evolution-frontend-review/final,examples/governed-incident-command/generated-skill,0.094,clear +frontend-review,note-cleanup,examples/evolution-frontend-review/final,examples/simple-note-cleanup/generated-skill,0.053,clear +frontend-review,frontend-review,examples/evolution-frontend-review/final,examples/team-frontend-review/generated-skill,0.64,collision +frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/final,geo-ranking-article-generator,0.036,clear +frontend-review,invalid-governance-skill,examples/evolution-frontend-review/final,tests/fixtures/governance_invalid_manifest,0.0,clear +frontend-review,broken-yaml-skill,examples/evolution-frontend-review/final,tests/fixtures/package_invalid_yaml,0.0,clear +frontend-review,broken-skill,examples/evolution-frontend-review/final,tests/fixtures/package_missing_interface_field,0.0,clear +frontend-review,frontend-review,examples/evolution-frontend-review/v0,examples/evolution-frontend-review/v1,0.136,clear +frontend-review,incident-command-governor,examples/evolution-frontend-review/v0,examples/governed-incident-command/generated-skill,0.056,clear +frontend-review,note-cleanup,examples/evolution-frontend-review/v0,examples/simple-note-cleanup/generated-skill,0.0,clear +frontend-review,frontend-review,examples/evolution-frontend-review/v0,examples/team-frontend-review/generated-skill,0.136,clear +frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/v0,geo-ranking-article-generator,0.0,clear +frontend-review,invalid-governance-skill,examples/evolution-frontend-review/v0,tests/fixtures/governance_invalid_manifest,0.0,clear +frontend-review,broken-yaml-skill,examples/evolution-frontend-review/v0,tests/fixtures/package_invalid_yaml,0.0,clear +frontend-review,broken-skill,examples/evolution-frontend-review/v0,tests/fixtures/package_missing_interface_field,0.0,clear +frontend-review,incident-command-governor,examples/evolution-frontend-review/v1,examples/governed-incident-command/generated-skill,0.097,clear +frontend-review,note-cleanup,examples/evolution-frontend-review/v1,examples/simple-note-cleanup/generated-skill,0.054,clear +frontend-review,frontend-review,examples/evolution-frontend-review/v1,examples/team-frontend-review/generated-skill,0.538,collision +frontend-review,geo-ranking-article-generator,examples/evolution-frontend-review/v1,geo-ranking-article-generator,0.037,clear +frontend-review,invalid-governance-skill,examples/evolution-frontend-review/v1,tests/fixtures/governance_invalid_manifest,0.0,clear +frontend-review,broken-yaml-skill,examples/evolution-frontend-review/v1,tests/fixtures/package_invalid_yaml,0.0,clear +frontend-review,broken-skill,examples/evolution-frontend-review/v1,tests/fixtures/package_missing_interface_field,0.0,clear +incident-command-governor,note-cleanup,examples/governed-incident-command/generated-skill,examples/simple-note-cleanup/generated-skill,0.065,clear +incident-command-governor,frontend-review,examples/governed-incident-command/generated-skill,examples/team-frontend-review/generated-skill,0.133,clear +incident-command-governor,geo-ranking-article-generator,examples/governed-incident-command/generated-skill,geo-ranking-article-generator,0.042,clear +incident-command-governor,invalid-governance-skill,examples/governed-incident-command/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear +incident-command-governor,broken-yaml-skill,examples/governed-incident-command/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear +incident-command-governor,broken-skill,examples/governed-incident-command/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear +note-cleanup,frontend-review,examples/simple-note-cleanup/generated-skill,examples/team-frontend-review/generated-skill,0.054,clear +note-cleanup,geo-ranking-article-generator,examples/simple-note-cleanup/generated-skill,geo-ranking-article-generator,0.038,clear +note-cleanup,invalid-governance-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear +note-cleanup,broken-yaml-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear +note-cleanup,broken-skill,examples/simple-note-cleanup/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear +frontend-review,geo-ranking-article-generator,examples/team-frontend-review/generated-skill,geo-ranking-article-generator,0.037,clear +frontend-review,invalid-governance-skill,examples/team-frontend-review/generated-skill,tests/fixtures/governance_invalid_manifest,0.0,clear +frontend-review,broken-yaml-skill,examples/team-frontend-review/generated-skill,tests/fixtures/package_invalid_yaml,0.0,clear +frontend-review,broken-skill,examples/team-frontend-review/generated-skill,tests/fixtures/package_missing_interface_field,0.0,clear +geo-ranking-article-generator,invalid-governance-skill,geo-ranking-article-generator,tests/fixtures/governance_invalid_manifest,0.0,clear +geo-ranking-article-generator,broken-yaml-skill,geo-ranking-article-generator,tests/fixtures/package_invalid_yaml,0.0,clear +geo-ranking-article-generator,broken-skill,geo-ranking-article-generator,tests/fixtures/package_missing_interface_field,0.0,clear +invalid-governance-skill,broken-yaml-skill,tests/fixtures/governance_invalid_manifest,tests/fixtures/package_invalid_yaml,0.111,clear +invalid-governance-skill,broken-skill,tests/fixtures/governance_invalid_manifest,tests/fixtures/package_missing_interface_field,0.125,clear +broken-yaml-skill,broken-skill,tests/fixtures/package_invalid_yaml,tests/fixtures/package_missing_interface_field,0.833,collision diff --git a/skill_atlas/stale_skills.json b/skill_atlas/stale_skills.json new file mode 100644 index 00000000..14361a6d --- /dev/null +++ b/skill_atlas/stale_skills.json @@ -0,0 +1,54 @@ +[ + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/final", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v0", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/evolution-frontend-review/v1", + "reason": "missing updated_at" + }, + { + "name": "incident-command-governor", + "path": "examples/governed-incident-command/generated-skill", + "reason": "review overdue by cadence monthly", + "age_days": 74, + "allowed_days": 31 + }, + { + "name": "note-cleanup", + "path": "examples/simple-note-cleanup/generated-skill", + "reason": "missing updated_at" + }, + { + "name": "frontend-review", + "path": "examples/team-frontend-review/generated-skill", + "reason": "missing updated_at" + }, + { + "name": "geo-ranking-article-generator", + "path": "geo-ranking-article-generator", + "reason": "missing updated_at" + }, + { + "name": "invalid-governance-skill", + "path": "tests/fixtures/governance_invalid_manifest", + "reason": "missing updated_at" + }, + { + "name": "broken-yaml-skill", + "path": "tests/fixtures/package_invalid_yaml", + "reason": "missing updated_at" + }, + { + "name": "broken-skill", + "path": "tests/fixtures/package_missing_interface_field", + "reason": "missing updated_at" + } +] diff --git a/tests/verify_conformance_suite.py b/tests/verify_conformance_suite.py new file mode 100644 index 00000000..5039ee5a --- /dev/null +++ b/tests/verify_conformance_suite.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "run_conformance_suite.py" +EXPORT_IR = ROOT / "scripts" / "export_skill_ir.py" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_conformance" + tmp_root.mkdir(parents=True, exist_ok=True) + output_json = tmp_root / "conformance_matrix.json" + output_md = tmp_root / "conformance_matrix.md" + subprocess.run( + [ + sys.executable, + str(EXPORT_IR), + str(ROOT), + "--output-json", + str(ROOT / "skill-ir" / "examples" / "yao-meta-skill.json"), + ], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["ok"], payload + assert payload["summary"]["target_count"] == 5, payload + assert payload["summary"]["pass_count"] == 5, payload + assert {item["target"] for item in payload["targets"]} == {"openai", "claude", "agent-skills", "vscode", "generic"}, payload + assert output_json.exists(), output_json + assert output_md.exists(), output_md + assert "Runtime Conformance Matrix" in output_md.read_text(encoding="utf-8") + + broken = tmp_root / "broken-skill" + (broken / "agents").mkdir(parents=True, exist_ok=True) + (broken / "reports").mkdir(parents=True, exist_ok=True) + (broken / "SKILL.md").write_text("---\nname: broken-skill\n---\n\n# Broken\n", encoding="utf-8") + (broken / "manifest.json").write_text(json.dumps({"name": "broken-skill"}, indent=2), encoding="utf-8") + (broken / "agents" / "interface.yaml").write_text("interface: {}\ncompatibility: {}\n", encoding="utf-8") + broken_proc = subprocess.run( + [sys.executable, str(SCRIPT), str(broken), "--target", "generic"], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert broken_proc.returncode == 2, broken_proc.stdout + broken_payload = json.loads(broken_proc.stdout) + assert not broken_payload["ok"], broken_payload + assert broken_payload["targets"][0]["failures"], broken_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_github_benchmark_scan.py b/tests/verify_github_benchmark_scan.py index 85b7ef43..98245cd5 100644 --- a/tests/verify_github_benchmark_scan.py +++ b/tests/verify_github_benchmark_scan.py @@ -48,6 +48,43 @@ def main() -> None: assert "anthropics/skills" in markdown, markdown[:800] assert "Cross-Repo Borrow Recommendations" in markdown, markdown[:1400] + empty_fixture = tmp_root / "empty_fixture" + empty_fixture.mkdir(parents=True, exist_ok=True) + (empty_fixture / "bundle.json").write_text( + json.dumps({"search_items": [], "readmes": {}}, ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + empty_skill_dir = tmp_root / "benchmark-empty-demo" + (empty_skill_dir / "reports").mkdir(parents=True, exist_ok=True) + (empty_skill_dir / "SKILL.md").write_text( + "---\nname: benchmark-empty-demo\ndescription: Turn a vague idea into a reusable skill.\n---\n\n# Empty Benchmark Demo\n", + encoding="utf-8", + ) + empty_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(empty_skill_dir), + "--query", + "unlikely benchmark query", + "--fixture-dir", + str(empty_fixture), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + empty_payload = json.loads(empty_proc.stdout) + assert empty_payload["ok"], empty_payload + assert empty_payload["repositories"] == [], empty_payload + assert empty_payload["cross_repo"]["borrow"] == [], empty_payload + assert empty_payload["external_references"] == [], empty_payload + assert empty_payload["borrow_prompt"].startswith("No benchmark suggestions"), empty_payload + empty_markdown = Path(empty_payload["artifacts"]["markdown"]).read_text(encoding="utf-8") + assert "No benchmark repositories were collected." in empty_markdown, empty_markdown[:600] + assert "Review the three benchmark objects" not in empty_markdown, empty_markdown[:600] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_output_eval_lab.py b/tests/verify_output_eval_lab.py new file mode 100644 index 00000000..71661b23 --- /dev/null +++ b/tests/verify_output_eval_lab.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "run_output_eval.py" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_output_eval" + tmp_root.mkdir(parents=True, exist_ok=True) + output_json = tmp_root / "output_quality_scorecard.json" + output_md = tmp_root / "output_quality_scorecard.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["ok"], payload + assert payload["summary"]["case_count"] == 3, payload + assert payload["summary"]["with_skill_pass_rate"] > payload["summary"]["baseline_pass_rate"], payload + assert payload["summary"]["delta"] > 0, payload + assert payload["summary"]["gate_pass"], payload + assert output_json.exists(), output_json + assert output_md.exists(), output_md + markdown = output_md.read_text(encoding="utf-8") + assert "Output Quality Scorecard" in markdown, markdown[:400] + assert "with-skill" in markdown, markdown[:600] + assert "Failure Taxonomy" in markdown, markdown[:1200] + + invalid_cases = tmp_root / "invalid_cases.jsonl" + invalid_cases.write_text(json.dumps({"id": "bad", "prompt": "x"}) + "\n", encoding="utf-8") + invalid_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(invalid_cases), + "--output-json", + str(tmp_root / "invalid.json"), + "--output-md", + str(tmp_root / "invalid.md"), + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert invalid_proc.returncode == 2, invalid_proc.stdout + invalid_payload = json.loads(invalid_proc.stdout) + assert not invalid_payload["ok"], invalid_payload + assert invalid_payload["failures"], invalid_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_atlas.py b/tests/verify_skill_atlas.py new file mode 100644 index 00000000..42fadf06 --- /dev/null +++ b/tests/verify_skill_atlas.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "build_skill_atlas.py" + + +def write_skill( + root: Path, + name: str, + description: str, + manifest: dict, + shared_reference: bool = False, +) -> Path: + skill_dir = root / name + skill_dir.mkdir(parents=True, exist_ok=True) + (skill_dir / "SKILL.md").write_text( + f"""--- +name: {name} +description: {description} +--- + +# {name} + +## Workflow + +1. Understand the request. +2. Produce the requested artifact. +3. Review the output. +""", + encoding="utf-8", + ) + (skill_dir / "manifest.json").write_text(json.dumps(manifest, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + if shared_reference: + references = skill_dir / "references" + references.mkdir(parents=True, exist_ok=True) + (references / "release-method.md").write_text("# Release Method\n", encoding="utf-8") + return skill_dir + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_skill_atlas" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + + write_skill( + tmp_root, + "alpha-release-skill", + "Use when turning release notes and changelog snippets into a reusable release brief for teams.", + { + "name": "alpha-release-skill", + "owner": "Release Team", + "maturity_tier": "production", + "review_cadence": "monthly", + "updated_at": "2025-01-01", + "target_platforms": ["openai", "generic"], + }, + shared_reference=True, + ) + write_skill( + tmp_root, + "beta-release-skill", + "Use when turning changelog notes and release snippets into reusable release briefs for team review.", + { + "name": "beta-release-skill", + "maturity_tier": "production", + "updated_at": "2026-06-01", + "target_platforms": ["openai"], + }, + shared_reference=True, + ) + write_skill( + tmp_root, + "gamma-doc-skill", + "Use when cleaning messy documentation notes into a concise internal knowledge-base article.", + { + "name": "gamma-doc-skill", + "owner": "Docs Team", + "maturity_tier": "scaffold", + "review_cadence": "quarterly", + "updated_at": "2026-05-01", + "target_platforms": ["generic"], + }, + ) + evals = tmp_root / "evals" + evals.mkdir() + (evals / "failure-cases.md").write_text( + "- no_route: repeated pricing approval prompts are missed by the current library.\n", + encoding="utf-8", + ) + + output_dir = tmp_root / "atlas" + report_html = tmp_root / "skill_atlas.html" + report_json = tmp_root / "skill_atlas.json" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--workspace-root", + str(tmp_root), + "--output-dir", + str(output_dir), + "--report-html", + str(report_html), + "--report-json", + str(report_json), + "--today", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert proc.returncode == 0, proc.stderr or proc.stdout + payload = json.loads(proc.stdout) + summary = payload["summary"] + assert summary["skill_count"] == 3, summary + assert summary["route_collision_count"] >= 1, summary + assert summary["owner_gap_count"] >= 1, summary + assert summary["stale_count"] >= 1, summary + assert summary["shared_resource_count"] >= 1, summary + assert summary["no_route_opportunity_count"] >= 1, summary + assert (output_dir / "catalog.json").exists(), output_dir + assert (output_dir / "route_overlap_matrix.csv").exists(), output_dir + assert (output_dir / "dependency_graph.json").exists(), output_dir + assert (output_dir / "stale_skills.json").exists(), output_dir + assert (output_dir / "owner_review_gaps.json").exists(), output_dir + assert (output_dir / "no_route_opportunities.json").exists(), output_dir + html = report_html.read_text(encoding="utf-8") + assert "Skill Atlas" in html, html[:1000] + assert "Route Collisions" in html, html[:3000] + assert "No-Route Opportunities" in html, html[:3000] + catalog = json.loads((output_dir / "catalog.json").read_text(encoding="utf-8")) + assert catalog["summary"]["skill_count"] == 3, catalog + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_ir.py b/tests/verify_skill_ir.py new file mode 100644 index 00000000..c81fb01d --- /dev/null +++ b/tests/verify_skill_ir.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "export_skill_ir.py" + + +def main() -> None: + output_path = ROOT / "tests" / "tmp_skill_ir" / "yao-meta-skill.json" + output_path.parent.mkdir(parents=True, exist_ok=True) + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_path), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + result = json.loads(proc.stdout) + assert result["ok"], result + assert result["summary"]["name"] == "yao-meta-skill", result + assert result["summary"]["trigger_samples"] >= 3, result + assert output_path.exists(), output_path + + payload = json.loads(output_path.read_text(encoding="utf-8")) + assert payload["schema_version"] == "2.0.0", payload + assert payload["name"] == "yao-meta-skill", payload + assert payload["job_to_be_done"], payload + assert payload["trigger_surface"]["description"], payload + assert payload["trigger_surface"]["should_trigger"], payload + assert payload["trigger_surface"]["should_not_trigger"], payload + assert payload["trigger_surface"]["edge_cases"], payload + assert payload["workflow"]["steps"], payload + assert payload["workflow"]["decision_points"], payload + assert payload["resources"]["references"], payload + assert payload["resources"]["scripts"], payload + assert payload["resources"]["reports"], payload + assert "evals/trigger_cases.json" in payload["eval_plan"]["trigger"], payload["eval_plan"] + assert payload["risk"]["trust_boundary"] == "external", payload + assert payload["governance"]["maturity"] == "governed", payload + assert "openai" in payload["targets"], payload + + validate_proc = subprocess.run( + [sys.executable, str(SCRIPT), str(ROOT), "--validate-only"], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + validate_result = json.loads(validate_proc.stdout) + assert validate_result["ok"], validate_result + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_skill_overview.py b/tests/verify_skill_overview.py index da987986..da16ff35 100644 --- a/tests/verify_skill_overview.py +++ b/tests/verify_skill_overview.py @@ -78,6 +78,16 @@ def main() -> None: "report_contract", } assert expected_v2_keys.issubset(overview_json.keys()), overview_json.keys() + assert "reports/skill-ir.json" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/output_quality_scorecard.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/skill_atlas.html" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert overview_json["skill_ir"]["schema_version"] in {"", "2.0.0"}, overview_json.get("skill_ir") + assert "output_quality" in overview_json, overview_json.keys() + assert "runtime_conformance" in overview_json, overview_json.keys() + assert "trust_security" in overview_json, overview_json.keys() + assert "skill_atlas" in overview_json, overview_json.keys() assert [item["title"] for item in overview_json["iteration_roadmap"]["items"]] == [ item["title"] for item in directions_json["directions"] ], { @@ -130,6 +140,7 @@ def main() -> None: assert "执行流程" in report_html, report_html[:5000] assert "调用方式" in report_html, report_html[:5000] assert "证据不足" in report_html or "证据充分" in report_html, report_html[:8000] + assert "原始说明可切换到英文查看；默认中文报告保留结论与结构说明。" not in report_html, report_html[:12000] assert "理解用户请求" in report_html, report_html[:5000] assert overview_json["logic_steps"][0] in report_html, overview_json.get("logic_steps") assert overview_json["usage_steps"][0] in report_html, overview_json.get("usage_steps") diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py new file mode 100644 index 00000000..a9cec4ef --- /dev/null +++ b/tests/verify_trust_check.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "trust_check.py" + + +INTERFACE = """interface: + display_name: "Trust Demo" + short_description: "Trust check demo" + default_prompt: "Use trust demo." +compatibility: + canonical_format: "agent-skills" + adapter_targets: + - "generic" + activation: + mode: "manual" + paths: [] + execution: + context: "inline" + shell: "bash" + trust: + source_tier: "local" + remote_inline_execution: "forbid" + remote_metadata_policy: "allow-metadata-only" + degradation: + generic: "neutral-source" +""" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_trust" + tmp_root.mkdir(parents=True, exist_ok=True) + output_json = tmp_root / "security_trust_report.json" + output_md = tmp_root / "security_trust_report.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["ok"], payload + assert payload["summary"]["secret_findings"] == 0, payload + assert payload["summary"]["package_sha256"], payload + assert output_json.exists(), output_json + assert output_md.exists(), output_md + assert "Security Trust Report" in output_md.read_text(encoding="utf-8") + + secret_skill = tmp_root / "secret-skill" + (secret_skill / "agents").mkdir(parents=True, exist_ok=True) + (secret_skill / "scripts").mkdir(parents=True, exist_ok=True) + (secret_skill / "SKILL.md").write_text( + "---\nname: secret-skill\ndescription: Secret demo.\n---\n\n# Secret\n", + encoding="utf-8", + ) + (secret_skill / "agents" / "interface.yaml").write_text(INTERFACE, encoding="utf-8") + (secret_skill / "scripts" / "leaky.py").write_text( + "TOKEN = 'ghp_1234567890abcdefghijklmnopqrstuv'\n", + encoding="utf-8", + ) + secret_proc = subprocess.run( + [sys.executable, str(SCRIPT), str(secret_skill)], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert secret_proc.returncode == 2, secret_proc.stdout + secret_payload = json.loads(secret_proc.stdout) + assert not secret_payload["ok"], secret_payload + assert secret_payload["summary"]["secret_findings"] == 1, secret_payload + assert secret_payload["secrets"][0]["type"] == "github_token", secret_payload + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 44a30bfb..1673b286 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -52,6 +52,7 @@ def main() -> None: assert (created / "reports" / "artifact-design-profile.md").exists(), created assert (created / "reports" / "prompt-quality-profile.md").exists(), created assert (created / "reports" / "system-model.md").exists(), created + assert (created / "reports" / "skill-ir.json").exists(), created assert (created / "reports" / "iteration-directions.md").exists(), created assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created init_report_view = init_result["payload"]["report_view"] @@ -61,6 +62,9 @@ def main() -> None: assert "概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view assert "默认使用中文简体" in init_report_view["message"], init_report_view assert "切换英文版" in init_report_view["message"], init_report_view + init_skill_ir = init_result["payload"]["skill_ir"] + assert init_skill_ir["name"] == "cli-demo-skill", init_skill_ir + assert init_skill_ir["trigger_samples"] >= 1, init_skill_ir quickstart_result = run( "quickstart", @@ -219,6 +223,52 @@ def main() -> None: assert directions_result["ok"], directions_result assert directions_result["payload"]["artifacts"]["markdown"].endswith("reports/iteration-directions.md"), directions_result + skill_ir_result = run("skill-ir", str(created)) + assert skill_ir_result["ok"], skill_ir_result + assert skill_ir_result["payload"]["artifacts"]["json"].endswith("reports/skill-ir.json"), skill_ir_result + created_skill_ir = json.loads((created / "reports" / "skill-ir.json").read_text(encoding="utf-8")) + assert created_skill_ir["schema_version"] == "2.0.0", created_skill_ir + assert created_skill_ir["trigger_surface"]["description"], created_skill_ir + + output_eval_result = run( + "output-eval", + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(created / "reports" / "output_quality_scorecard.json"), + "--output-md", + str(created / "reports" / "output_quality_scorecard.md"), + ) + assert output_eval_result["ok"], output_eval_result + assert output_eval_result["payload"]["summary"]["with_skill_pass_rate"] > output_eval_result["payload"]["summary"]["baseline_pass_rate"], output_eval_result + + conformance_result = run("conformance", str(created)) + assert conformance_result["ok"], conformance_result + assert conformance_result["payload"]["summary"]["target_count"] == 5, conformance_result + assert conformance_result["payload"]["artifacts"]["markdown"].endswith("reports/conformance_matrix.md"), conformance_result + + trust_result = run("trust", str(created)) + assert trust_result["ok"], trust_result + assert trust_result["payload"]["summary"]["secret_findings"] == 0, trust_result + assert trust_result["payload"]["artifacts"]["markdown"].endswith("reports/security_trust_report.md"), trust_result + + atlas_result = run( + "skill-atlas", + "--workspace-root", + str(tmp_root), + "--output-dir", + str(tmp_root / "skill_atlas"), + "--report-html", + str(tmp_root / "skill_atlas.html"), + "--report-json", + str(tmp_root / "skill_atlas.json"), + "--today", + "2026-06-13", + ) + assert atlas_result["ok"], atlas_result + assert atlas_result["payload"]["summary"]["skill_count"] >= 2, atlas_result + assert atlas_result["payload"]["artifacts"]["report_html"].endswith("skill_atlas.html"), atlas_result + feedback_result = run( "feedback", str(created),