frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
6e2dfd48d080901f09b4e67bdf35b9500ec3dcec
yao-meta-skill/reports/review-viewer.json
T
yaojingang 27a5c12008 Use package manifest name for archives
2026-06-15 22:15:53 +08:00

14564 lines
518 KiB
JSON
Raw Blame History

{
"overview": {
"skill_summary": {
"name": "yao-meta-skill",
"title": "Yao Meta Skill",
"display_name": "Yao Meta Skill",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"maturity": "governed",
"updated_at": "2026-03-31",
"core_value": "把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。",
"audience": "Skill 作者、复用团队和后续 reviewer。",
"deliverables": [
"SKILL.md",
"agents/interface.yaml",
"reports/skill-ir.json",
"reports/compiled_targets.md",
"reports/output_quality_scorecard.md",
"reports/output_execution_runs.md",
"reports/output_blind_review_pack.md",
"reports/output_review_kit.md",
"reports/output_review_adjudication.md",
"reports/benchmark_reproducibility.md",
"reports/output_blind_answer_key.json",
"reports/conformance_matrix.md",
"reports/security_trust_report.md",
"reports/runtime_permission_probes.md",
"reports/skill_atlas.html",
"reports/registry_audit.md",
"reports/package_verification.md",
"reports/install_simulation.md",
"reports/upgrade_check.md",
"reports/adoption_drift_report.md",
"reports/review_waivers.md",
"reports/world_class_evidence_plan.md",
"reports/world_class_evidence_ledger.md",
"reports/review_annotations.md",
"reports/review-studio.html",
"reports/skill-interpretation.html",
"reports/skill-overview.html"
],
"flow": [
"输入材料",
"Skill 包体",
"可复用能力"
]
},
"scorecard": {
"completeness_score": {
"label": "完整度",
"score": 100,
"reasons": [
"SKILL.md 已存在，是 Skill 的入口。",
"README.md 已存在，便于人工阅读。",
"agents/interface.yaml 已存在，便于跨平台适配。",
"manifest.json 已存在，生命周期信息可追踪。",
"reports/ 已存在，生成证据可以随包体迁移。"
]
},
"trigger_score": {
"label": "触发清晰",
"score": 100,
"reasons": [
"frontmatter description 已存在，具备基础路由面。",
"description 有足够长度说明任务边界。",
"description 已包含使用场景或排除边界信号。",
"evals/ 已存在，可承载触发样例或质量检查。",
"intent-confidence 报告已生成，可辅助判断触发稳定性。"
]
},
"evidence_score": {
"label": "证据充分",
"score": 100,
"reasons": [
"已生成 20 / 20 类报告证据。",
"skill-ir.json 已存在。",
"compiled_targets.json 已存在。",
"intent-dialogue.json 已存在。"
]
},
"maintainability_score": {
"label": "可维护性",
"score": 100,
"reasons": [
"SKILL.md 约 356 个词/字。",
"入口文件保持克制，可维护性较好。",
"references/ 已承载扩展指导。",
"scripts/ 已承载确定性逻辑。",
"evals/ 已承载可迁移检查。"
]
},
"portability_score": {
"label": "可迁移性",
"score": 100,
"reasons": [
"agents/interface.yaml 已存在。",
"manifest.json 已存在。",
"目标平台或 adapter target 已声明。",
"入口文件未发现明显私有绝对路径。"
]
},
"context_cost": {
"label": "上下文成本",
"score": 42,
"reasons": [
"入口约 356 个词/字，references 约 16045 个词/字。",
"分数越高代表上下文成本越低。",
"上下文成本偏高，建议压缩入口或拆分 references。"
]
}
},
"capability_profile": {
"archetype": "governed",
"task_family": "Execution operation",
"maturity": "governed",
"trigger_strength": "手动触发 + description 路由",
"reuse_scope": "跨平台",
"matrix": {
"execution_certainty": 72,
"knowledge_density": 80
}
},
"principle_model": {
"nodes": [
{
"title": "意图澄清",
"body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse."
},
{
"title": "边界路由",
"body": "用 frontmatter description 决定是否触发，并写明相邻非目标。"
},
{
"title": "资产分层",
"body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。"
},
{
"title": "证据回路",
"body": "Ask only the highest-leverage clarification before adding package weight."
},
{
"title": "漂移观察",
"body": "Add near-neighbor exclusions and route evals before expanding workflow steps."
},
{
"title": "杠杆升级",
"body": "Name the recurring job, expected input, output, and strongest non-goal in compact language."
}
],
"layers": [
"入口层",
"参考层",
"脚本层",
"评估层",
"报告层"
]
},
"contract_boundary": {
"trigger": {
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"activation": "manual",
"execution": "inline",
"shell": "bash"
},
"inputs": [
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"outputs": [
"A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"结构化 Skill 目录，共 8 类关键资产。"
],
"should_trigger": [
"把重复流程沉淀为可复用的 agent skill。",
"把分散提示词、对话记录或操作规范整理为稳定能力。",
"团队复用前，需要明确触发边界、质量证据和维护责任。"
],
"should_not_trigger": [
"只需要一次性回答、没有复用价值的临时请求。",
"要求直接执行相邻任务，而不是沉淀或使用这个 Skill。",
"缺少必要事实且用户不允许澄清的场景。"
],
"boundary_cards": [
{
"label": "Owned",
"body": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
},
{
"label": "Adjacent",
"body": "相邻任务需要先确认是否应转为独立 Skill。"
},
{
"label": "Excluded",
"body": "不替代人工事实核查，也不静默扩大职责。"
}
]
},
"quality_review": {
"strengths": [
"触发面保持精简，并锚定在 frontmatter description。",
"已生成 Skill IR，核心语义可先于平台打包被审查和迁移。",
"已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。",
"已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。",
"已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。",
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。"
],
"gaps": [
"上下文成本需要补强：入口约 356 个词/字，references 约 16045 个词/字。"
],
"recommendations": [
"先改触发边界，再扩展工作流。",
"只把重复且稳定的步骤沉淀为脚本。",
"每次升级后重新生成报告并检查分数原因。"
],
"artifact_design": {
"design_system": "metric editorial",
"highlights": [
"Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
"Name the working directory and required inputs for commands.",
"Mark destructive, networked, or external side-effect operations.",
"Prefer the smallest runnable snippet over broad framework scaffolding."
]
},
"prompt_quality": {
"overall_quality_score": 89.0,
"highlights": [
"Primary prompt task family: Execution operation.",
"Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints",
"Completeness: 100/100.",
"Clarity: 85/100."
]
},
"system_model": {
"stability": {
"score": 100,
"band": "system-ready"
},
"highlights": [
"Stability: system-ready (100/100).",
"Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.",
"Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes."
]
}
},
"risk_governance": {
"risks": [
{
"name": "误触发风险",
"impact": 3,
"probability": 1,
"signal": "frontmatter description 已存在，具备基础路由面。",
"response": "先补证据和边界，再增加包体复杂度。"
},
{
"name": "输出漂移风险",
"impact": 2,
"probability": 1,
"signal": "已生成 20 / 20 类报告证据。",
"response": "先补证据和边界，再增加包体复杂度。"
},
{
"name": "证据不足风险",
"impact": 3,
"probability": 1,
"signal": "已生成 20 / 20 类报告证据。",
"response": "先补证据和边界，再增加包体复杂度。"
},
{
"name": "包体膨胀风险",
"impact": 2,
"probability": 1,
"signal": "SKILL.md 约 356 个词/字。",
"response": "先补证据和边界，再增加包体复杂度。"
},
{
"name": "跨平台迁移风险",
"impact": 3,
"probability": 1,
"signal": "agents/interface.yaml 已存在。",
"response": "先补证据和边界，再增加包体复杂度。"
}
],
"risk_families": [
{
"key": "markdown_readability",
"label": "Markdown readability",
"matched_keywords": [
"md",
"report",
"doc"
],
"score": 3,
"risks": [
"Tables can render as dense grids with weak hierarchy or poor mobile readability.",
"Long bullets can make the output look complete while hiding the actual decision logic.",
"Mixed heading levels can reduce scanability."
],
"constraints": [
"Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.",
"Keep table cells short and move explanations below the table.",
"Use heading levels consistently and keep each section anchored to a user-facing outcome."
],
"self_repair": [
"Preview whether each table still reads well when columns are narrow.",
"Convert any table with paragraph-length cells into bullets or cards."
]
},
{
"key": "citation_clutter",
"label": "Citation and footnote clutter",
"matched_keywords": [
"source",
"reference"
],
"score": 2,
"risks": [
"Footnote markers or dense citation notes can interrupt the reading flow.",
"Evidence can be over-attached to obvious statements and under-attached to risky claims.",
"Source notes may become more prominent than the tutorial itself."
],
"constraints": [
"Attach citations only to claims that need evidence, not to every sentence.",
"Group source notes at the end of a section when inline markers would hurt readability.",
"Keep the main sentence readable without requiring the reader to chase a footnote."
],
"self_repair": [
"Remove decorative citations that do not support a material claim.",
"Move repeated source explanations into one compact source note."
]
},
{
"key": "visual_capture",
"label": "Screenshot and visual capture",
"matched_keywords": [
"capture"
],
"score": 1,
"risks": [
"Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.",
"Missing screenshots can cause the skill to invent visual references instead of declaring the gap.",
"Image descriptions can omit the action-relevant region."
],
"constraints": [
"Never invent a screenshot; state when visual evidence is missing.",
"Record the source, viewport, and crop intent for any screenshot-dependent output.",
"Describe what the reader should inspect in the image, not just that an image exists."
],
"self_repair": [
"Check that every screenshot reference points to a real provided or generated asset.",
"Reword any visual instruction that depends on an unseen screen state."
]
},
{
"key": "code_or_command_safety",
"label": "Code and command safety",
"matched_keywords": [
"script"
],
"score": 1,
"risks": [
"Commands can omit environment assumptions, working directory, or rollback notes.",
"Code snippets can look runnable while missing required inputs.",
"Error handling can be either absent or over-engineered."
],
"constraints": [
"Name the working directory, required inputs, and expected output for each command.",
"Mark destructive or external side-effect operations explicitly.",
"Prefer the smallest runnable snippet over broad framework code."
],
"self_repair": [
"Scan each command for cwd, input, output, and side-effect assumptions.",
"Remove speculative error handling that is not tied to a real failure mode."
]
},
{
"key": "tone_and_specificity",
"label": "Tone and specificity",
"matched_keywords": [
"summary"
],
"score": 1,
"risks": [
"Headings and summaries can drift into generic, interchangeable language.",
"The output can sound polished but lose the user's actual taste, audience, or scenario.",
"Strong claims can appear without examples or constraints."
],
"constraints": [
"Anchor titles and summaries in the user's audience, object, and concrete outcome.",
"Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.",
"Preserve one distinctive phrase, constraint, or standard from the user's input."
],
"self_repair": [
"Replace generic title candidates with scenario-specific alternatives.",
"Delete any polished sentence that could fit almost any project unchanged."
]
}
],
"human_judgment_boundary": [
"Ask one focused clarification when the real job, output, or exclusion boundary is unclear.",
"Escalate visible tradeoffs when benchmark patterns conflict with local privacy, naming, or governance constraints.",
"Do not silently broaden the skill into adjacent jobs just because the examples are nearby."
]
},
"world_class_readiness": {
"ready": false,
"decision": "evidence-pending",
"entry_count": 4,
"pending_count": 4,
"accepted_count": 0,
"external_pending_count": 3,
"human_pending_count": 1,
"source_check_count": 13,
"source_pass_count": 6,
"conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。",
"conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.",
"entries": [
{
"key": "provider-holdout",
"label_zh": "提供商留出",
"label_en": "Provider Holdout",
"category": "external",
"category_zh": "外部证据",
"category_en": "External evidence",
"status": "pending",
"summary_zh": "缺少真实 provider 模型运行和 token metadata。",
"summary_en": "Missing a real provider model run and token metadata.",
"blocked_checks": [
"Provider model run",
"Token usage observed"
]
},
{
"key": "human-adjudication",
"label_zh": "人工盲评",
"label_en": "Human Adjudication",
"category": "human",
"category_zh": "人工证据",
"category_en": "Human evidence",
"status": "pending",
"summary_zh": "盲评 pair 仍待真实 reviewer 决策。",
"summary_en": "Blind-review pairs still need real reviewer decisions.",
"blocked_checks": [
"No pending decisions",
"Judgments complete"
]
},
{
"key": "native-permission-enforcement",
"label_zh": "原生权限",
"label_en": "Native Permission",
"category": "external",
"category_zh": "外部证据",
"category_en": "External evidence",
"status": "pending",
"summary_zh": "原生 runtime enforcement 仍待目标客户端或外部安装器证明。",
"summary_en": "Native runtime enforcement still needs target-client or external-installer proof.",
"blocked_checks": [
"Native enforcement"
]
},
{
"key": "native-client-telemetry",
"label_zh": "原生遥测",
"label_en": "Native Telemetry",
"category": "external",
"category_zh": "外部证据",
"category_en": "External evidence",
"status": "pending",
"summary_zh": "真实外部客户端 metadata-only 事件仍未导入。",
"summary_en": "Real external-client metadata-only events have not been imported yet.",
"blocked_checks": [
"External events",
"Adoption sample"
]
}
]
},
"package_assets": {
"entries": [
{
"path": "SKILL.md",
"label": "Skill entrypoint",
"kind": "file",
"file_count": 1
},
{
"path": "README.md",
"label": "Human-readable usage guide",
"kind": "file",
"file_count": 1
},
{
"path": "agents/interface.yaml",
"label": "Neutral interface metadata",
"kind": "file",
"file_count": 1
},
{
"path": "manifest.json",
"label": "Lifecycle and portability metadata",
"kind": "file",
"file_count": 1
},
{
"path": "references",
"label": "Extended guidance and reusable notes",
"kind": "folder",
"file_count": 34
},
{
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 107
},
{
"path": "evals",
"label": "Trigger and quality checks",
"kind": "folder",
"file_count": 29
},
{
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 215
}
],
"file_count": 389,
"folder_count": 4,
"distribution": [
{
"label": "SKILL.md",
"value": 1
},
{
"label": "README.md",
"value": 1
},
{
"label": "agents/interface.yaml",
"value": 1
},
{
"label": "manifest.json",
"value": 1
},
{
"label": "references",
"value": 34
},
{
"label": "scripts",
"value": 107
},
{
"label": "evals",
"value": 29
},
{
"label": "reports",
"value": 215
}
]
},
"iteration_roadmap": {
"items": [
{
"title": "补齐世界证据",
"why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。",
"actions": [
"补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。",
"补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。",
"继续补齐剩余 2 项外部/人工证据，并保持 claim guard 为 pending 状态。"
],
"unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。",
"source": "world_class_evidence_ledger"
},
{
"title": "Borrow one proven pattern on purpose",
"why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.",
"actions": [
"Read the strongest pattern from obra/superpowers.",
"Decide whether to borrow method, structure, execution, or portability, but only one of them first.",
"Record what you will not borrow so the package stays light."
],
"unlocks": "A cleaner package shape with less accidental over-design."
},
{
"title": "Harden portability semantics",
"why": "The skill already signals reuse across environments, so contract clarity matters early.",
"actions": [
"Confirm activation mode, execution context, and trust assumptions.",
"Add or review degradation strategy for non-native targets.",
"Package the skill once to verify adapter expectations."
],
"unlocks": "Safer cross-environment reuse with less target drift."
}
]
},
"report_contract": {
"schema_version": "2.0",
"html_report": "reports/skill-overview.html",
"language": "zh-CN",
"default_language": "zh-CN",
"languages": [
"zh-CN",
"en"
],
"layout": "kami-white-audit-v2",
"nav_labels": [
"技能概述",
"总览指标",
"能力画像",
"原理结构",
"契约边界",
"质量评估",
"风险治理",
"包体资产",
"迭代路线"
],
"nav_labels_en": [
"Overview",
"Metrics",
"Profile",
"Principle",
"Contract",
"Quality",
"Risk",
"Assets",
"Roadmap"
]
},
"name": "yao-meta-skill",
"title": "Yao Meta Skill",
"display_name": "Yao Meta Skill",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"logic_steps": [
"Decide whether the request should become a skill and choose the lightest fit.",
"Capture job, output, exclusions, constraints, and standards.",
"Run reference scan: external benchmarks first, user references second, local fit third; surface only uncertainty or conflict.",
"Write the `description` early and test route quality before expanding the package.",
"Add output-risk, artifact-design, prompt-quality, and system-model reports only when they matter."
],
"usage_steps": [
"Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals.",
"Use this skill when the request matches: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
],
"package_map": [
{
"path": "SKILL.md",
"label": "Skill entrypoint",
"kind": "file",
"file_count": 1
},
{
"path": "README.md",
"label": "Human-readable usage guide",
"kind": "file",
"file_count": 1
},
{
"path": "agents/interface.yaml",
"label": "Neutral interface metadata",
"kind": "file",
"file_count": 1
},
{
"path": "manifest.json",
"label": "Lifecycle and portability metadata",
"kind": "file",
"file_count": 1
},
{
"path": "references",
"label": "Extended guidance and reusable notes",
"kind": "folder",
"file_count": 34
},
{
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 107
},
{
"path": "evals",
"label": "Trigger and quality checks",
"kind": "folder",
"file_count": 29
},
{
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 215
}
],
"strengths": [
"触发面保持精简，并锚定在 frontmatter description。",
"已生成 Skill IR，核心语义可先于平台打包被审查和迁移。",
"已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。",
"已生成 Output Eval Lab scorecard，可比较 with-skill 与 baseline 输出质量。",
"已生成 Output Execution Runs，可区分记录样本、命令执行和模型执行证据。",
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。"
],
"scenarios": [
"把重复流程沉淀为可复用的 agent skill。",
"把分散提示词、对话记录或操作规范整理为稳定能力。",
"团队复用前，需要明确触发边界、质量证据和维护责任。",
"用户说出类似需求时：Use $yao-meta-skill to turn my workflow or notes into a reusable skill with lean structure, clear triggering, and the right evals."
],
"trigger_contract": {
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"activation": "manual",
"execution": "inline",
"shell": "bash"
},
"io_contract": {
"inputs": [
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"outputs": [
"A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"结构化 Skill 目录，共 8 类关键资产。"
]
},
"principles": [
{
"title": "意图澄清",
"body": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse."
},
{
"title": "边界路由",
"body": "用 frontmatter description 决定是否触发，并写明相邻非目标。"
},
{
"title": "资产分层",
"body": "把入口、参考、脚本、评估和报告拆到各自目录，避免 SKILL.md 膨胀。"
},
{
"title": "证据回路",
"body": "Ask only the highest-leverage clarification before adding package weight."
},
{
"title": "漂移观察",
"body": "Add near-neighbor exclusions and route evals before expanding workflow steps."
},
{
"title": "杠杆升级",
"body": "Name the recurring job, expected input, output, and strongest non-goal in compact language."
}
],
"roadmap": [
{
"title": "补齐世界证据",
"why": "世界级证据仍有 4 项待补；公开完成态 claim 必须继续保持阻塞。",
"actions": [
"补齐提供商留出证据：缺少真实 provider 模型运行和 token metadata。",
"补齐人工盲评证据：盲评 pair 仍待真实 reviewer 决策。",
"继续补齐剩余 2 项外部/人工证据，并保持 claim guard 为 pending 状态。"
],
"unlocks": "全部外部/人工证据被 ledger 接受后，才能进入公开 world-class claim 复核。",
"source": "world_class_evidence_ledger"
},
{
"title": "Borrow one proven pattern on purpose",
"why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.",
"actions": [
"Read the strongest pattern from obra/superpowers.",
"Decide whether to borrow method, structure, execution, or portability, but only one of them first.",
"Record what you will not borrow so the package stays light."
],
"unlocks": "A cleaner package shape with less accidental over-design."
},
{
"title": "Harden portability semantics",
"why": "The skill already signals reuse across environments, so contract clarity matters early.",
"actions": [
"Confirm activation mode, execution context, and trust assumptions.",
"Add or review degradation strategy for non-native targets.",
"Package the skill once to verify adapter expectations."
],
"unlocks": "Safer cross-environment reuse with less target drift."
}
],
"cards": [],
"introduction": [
"这份报告用于快速理解新生成 Skill 的定位、原理、触发边界和交付内容。",
"先确认重复任务、真实输入形态和可交付输出，再决定是否继续加 references、scripts 或 evals。",
"如果需求仍然模糊，优先回到 intent dialogue 收紧边界，再扩展包体结构。"
],
"benchmark_highlights": [],
"skill_ir": {
"schema_version": "2.0.0",
"target_count": 5,
"trigger_samples": 8,
"output_eval_cases": 3
},
"compiled_targets": {
"ok": true,
"schema_version": "1.0",
"summary": {
"target_count": 5,
"pass_count": 5,
"warn_count": 0,
"block_count": 0,
"failure_count": 0,
"warning_count": 0
},
"targets": [
{
"target": "openai",
"status": "pass",
"adapter_mode": "metadata-adapter",
"degradation_strategy": "metadata-adapter",
"native_surface": "OpenAI-style interface metadata plus neutral Agent Skills source",
"permission_enforcement": "metadata-only",
"generated_files": [
"targets/openai/adapter.json",
"targets/openai/agents/openai.yaml"
],
"unsupported_features": [
"client-native script permission prompts are represented as permission contract metadata"
],
"warnings": []
},
{
"target": "claude",
"status": "pass",
"adapter_mode": "neutral-source-plus-adapter",
"degradation_strategy": "neutral-source-plus-adapter",
"native_surface": "Claude-compatible neutral source folder with adapter notes",
"permission_enforcement": "metadata-fallback",
"generated_files": [
"targets/claude/adapter.json",
"targets/claude/README.md"
],
"unsupported_features": [
"vendor-native metadata fields are carried as adapter JSON and README notes"
],
"warnings": []
},
{
"target": "generic",
"status": "pass",
"adapter_mode": "agent-skills-compatible",
"degradation_strategy": "neutral-source",
"native_surface": "Agent Skills compatible neutral package",
"permission_enforcement": "consumer-enforced-or-metadata-only",
"generated_files": [
"targets/generic/adapter.json"
],
"unsupported_features": [],
"warnings": []
},
{
"target": "agent-skills-compatible",
"status": "pass",
"adapter_mode": "neutral-agent-skills-source",
"degradation_strategy": "neutral-source",
"native_surface": "Agent Skills standard source tree",
"permission_enforcement": "consumer-enforced-or-metadata-only",
"generated_files": [
"SKILL.md",
"agents/interface.yaml"
],
"unsupported_features": [],
"warnings": []
},
{
"target": "vscode",
"status": "pass",
"adapter_mode": "vscode-agent-skills-adapter",
"degradation_strategy": "agent-skills-source-with-vscode-notes",
"native_surface": "VS Code/Copilot Agent Skills project or user scope",
"permission_enforcement": "client-or-workspace-trust",
"generated_files": [
"targets/vscode/adapter.json",
"targets/vscode/README.md"
],
"unsupported_features": [
"VS Code installation scope is documented but not installed by this compiler"
],
"warnings": []
}
],
"failures": [],
"warnings": []
},
"output_quality": {
"case_count": 5,
"file_backed_case_count": 1,
"near_neighbor_case_count": 1,
"boundary_case_count": 1,
"baseline_pass_rate": 0.0,
"with_skill_pass_rate": 100.0,
"delta": 100.0,
"regression_count": 0,
"gate_pass": true,
"failure_taxonomy": [],
"blind_pair_count": 5
},
"output_execution": {
"ok": true,
"summary": {
"case_count": 5,
"variant_run_count": 10,
"command_executed_count": 10,
"model_executed_count": 0,
"recorded_fixture_count": 0,
"timing_observed_count": 10,
"token_observed_count": 0,
"token_estimated_count": 10,
"baseline_pass_rate": 0.0,
"with_skill_pass_rate": 100.0,
"delta": 100.0,
"regression_count": 0,
"failure_count": 0,
"gate_pass": true
},
"runner": {
"mode": "command",
"command": [
"python3",
"scripts/local_output_eval_runner.py"
],
"timeout_seconds": 30.0
},
"failures": []
},
"output_blind_review": {
"summary": {
"pair_count": 5,
"answer_key_separate": true,
"with_skill_hidden_count": 5
},
"seed": "yao-output-eval-blind-v1",
"pair_count": 5,
"answer_key_separate": true
},
"output_review_kit": {
"ok": true,
"summary": {
"case_count": 5,
"ready_for_adjudication_count": 0,
"pending_decision_count": 5,
"invalid_decision_count": 0,
"reviewer_metadata_present": false,
"answer_key_hidden": true,
"answer_key_path_exposed": false,
"ready_to_run_adjudication": false,
"failure_count": 0
},
"artifacts": {
"reviewer_kit_json": "reports/output_review_kit.json",
"reviewer_kit_markdown": "reports/output_review_kit.md",
"reviewer_kit_html": "reports/output_review_kit.html",
"blind_pack_json": "reports/output_blind_review_pack.json",
"blind_pack_markdown": "reports/output_blind_review_pack.md",
"decisions": "reports/output_review_decisions.json",
"adjudication_json": "reports/output_review_adjudication.json",
"adjudication_markdown": "reports/output_review_adjudication.md"
},
"failures": []
},
"output_review_adjudication": {
"ok": true,
"summary": {
"pair_count": 5,
"judgment_count": 0,
"pending_count": 5,
"agreement_count": 0,
"disagreement_count": 0,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"pending_answer_hidden_count": 5,
"agreement_rate": null,
"needs_review": true,
"failure_count": 0,
"reviewer_checklist_count": 5,
"reviewer_checklist_pending_count": 5,
"reviewer_checklist_invalid_count": 0,
"reviewer_checklist_ready_count": 0
},
"reviewer": "",
"reviewed_at": "",
"failures": []
},
"benchmark_reproducibility": {
"ok": true,
"summary": {
"reproducibility_ready": true,
"release_lock_ready": false,
"methodology_complete": true,
"required_artifact_count": 24,
"missing_artifact_count": 0,
"evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab",
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc",
"output_case_count": 5,
"failure_disclosure_count": 3,
"command_count": 22,
"command_executed_count": 10,
"timing_observed_count": 10,
"model_executed_count": 0,
"token_observed_count": 0,
"human_review_complete": false,
"provider_evidence_complete": false,
"world_class_ready": false,
"world_class_open_gap_count": 4,
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"world_class_source_check_count": 13,
"world_class_source_pass_count": 6,
"world_class_source_blocked_count": 7,
"public_claim_ready": false,
"public_claim_blocker_count": 5,
"working_tree_dirty": true,
"changed_file_count": 38
},
"commit": "db1b24cbda5eb00f71b35846dff0f178625aa31d",
"missing_artifacts": [],
"limitations": [
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
"Pending blind-review decisions are visible but do not count as human adjudication.",
"World-class readiness remains false until external and human evidence gaps close."
]
},
"runtime_conformance": {
"target_count": 5,
"pass_count": 5,
"fail_count": 0
},
"runtime_permissions": {
"ok": true,
"summary": {
"target_count": 4,
"pass_count": 4,
"fail_count": 0,
"native_enforcement_count": 0,
"metadata_fallback_count": 4,
"residual_risk_count": 4,
"required_capability_count": 3,
"failure_count": 0,
"installer_enforcement_source_status": "present",
"installer_enforcement_target_count": 4,
"installer_enforcement_pass_count": 4,
"installer_permission_enforced_count": 12,
"installer_permission_failure_count": 0,
"installer_permission_capability_count": 3,
"world_class_native_evidence_ready": false,
"installer_enforcement_ready": true
},
"expected_capabilities": [
"file_write",
"network",
"subprocess"
],
"targets": [
{
"target": "openai",
"status": "pass",
"assurance": "metadata-fallback-explicit",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
},
{
"target": "claude",
"status": "pass",
"assurance": "metadata-fallback-explicit",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
},
{
"target": "generic",
"status": "pass",
"assurance": "metadata-fallback-explicit",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
},
{
"target": "vscode",
"status": "pass",
"assurance": "metadata-fallback-explicit",
"native_enforcement": false,
"metadata_fallback_explicit": true,
"residual_risks": [
"Client-native permission enforcement is not provided by this target; installer or operator must honor metadata."
]
}
],
"failures": []
},
"trust_security": {
"scanned_files": 194,
"script_count": 107,
"internal_module_count": 26,
"secret_findings": 0,
"dependency_files": [
"requirements-ci.txt"
],
"network_script_count": 3,
"network_policy_covered_count": 3,
"network_policy_missing_count": 0,
"file_write_script_count": 68,
"permission_required_count": 3,
"permission_approved_count": 3,
"permission_missing_count": 0,
"permission_invalid_count": 0,
"permission_expired_count": 0,
"help_smoke_checked_count": 81,
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 194,
"package_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306"
},
"skill_atlas": {
"skill_count": 12,
"actionable_skill_count": 1,
"route_collision_count": 5,
"actionable_route_collision_count": 0,
"owner_gap_count": 9,
"actionable_owner_gap_count": 0,
"stale_count": 10,
"actionable_stale_count": 0,
"shared_resource_count": 0,
"no_route_opportunity_count": 0,
"telemetry_report_count": 1,
"drift_signal_count": 0,
"actionable_drift_signal_count": 0,
"non_actionable_issue_count": 24
},
"registry_distribution": {
"ok": true,
"package": {
"schema_version": "2.0",
"name": "yao-meta-skill",
"version": "1.1.0",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"targets": [
"openai",
"claude",
"generic",
"agent-skills-compatible",
"vscode"
],
"maturity": "governed",
"owner": "Yao Team",
"review_cadence": "quarterly",
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc"
},
"compatibility": {
"openai": "pass",
"claude": "pass",
"agent-skills": "pass",
"vscode": "pass",
"generic": "pass",
"agent-skills-compatible": "pass"
},
"source": {
"skill_dir": ".",
"skill_ir": "skill-ir/examples/yao-meta-skill.json",
"ir_schema_version": "2.0.0",
"canonical_metadata": "agents/interface.yaml"
},
"artifacts": {
"overview_html": "reports/skill-overview.html",
"review_studio_html": "reports/review-studio.html",
"trust_report": "reports/security_trust_report.md",
"conformance_matrix": "reports/conformance_matrix.md",
"compiled_targets": "reports/compiled_targets.md",
"package_verification": "reports/package_verification.md",
"install_simulation": "reports/install_simulation.md",
"adoption_drift": "reports/adoption_drift_report.md",
"review_waivers": "reports/review_waivers.md",
"review_annotations": "reports/review_annotations.md",
"package_metadata": "registry/packages/yao-meta-skill.json"
},
"distribution": {
"archive_verified": true,
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
},
"generated_at": "2026-06-13"
},
"failures": [],
"warnings": []
},
"package_verification": {
"ok": true,
"summary": {
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc",
"archive_entry_count": 586,
"failure_count": 0,
"warning_count": 0
},
"failures": [],
"warnings": []
},
"install_simulation": {
"ok": true,
"summary": {
"archive_present": true,
"archive_entry_count": 607,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
"interface_loaded": true,
"adapter_count": 4,
"installer_permission_enforced_count": 12,
"installer_permission_failure_count": 0,
"permission_target_count": 4,
"permission_capability_count": 3,
"install_root_is_temp": false,
"failure_count": 0,
"warning_count": 0
},
"failures": [],
"warnings": []
},
"upgrade_check": {
"ok": true,
"summary": {
"declared_bump": "minor",
"recommended_bump": "minor",
"breaking_change_count": 0,
"added_target_count": 2,
"metadata_change_count": 1,
"checksum_change_count": 2,
"failure_count": 0,
"warning_count": 0
},
"upgrade_diff": {
"name_changed": false,
"added_targets": [
"agent-skills-compatible",
"vscode"
],
"removed_targets": [],
"compatibility_changes": [
{
"target": "agent-skills",
"from": "missing",
"to": "pass",
"direction": "improved"
},
{
"target": "agent-skills-compatible",
"from": "missing",
"to": "pass",
"direction": "improved"
},
{
"target": "vscode",
"from": "missing",
"to": "pass",
"direction": "improved"
}
],
"metadata_changes": [
{
"field": "description",
"from": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes.",
"to": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
}
],
"checksum_changes": [
{
"field": "archive_sha256",
"from": "",
"to": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306"
}
]
},
"release_notes": [
"Recommended version bump: minor.",
"Added targets: agent-skills-compatible, vscode.",
"Package or archive checksum changed; reviewers should verify package artifacts before release.",
"Package verification evidence: reports/package_verification.md."
],
"failures": [],
"warnings": []
},
"adoption_drift": {
"ok": true,
"summary": {
"event_count": 1,
"adoption_sample_count": 0,
"activation_count": 0,
"accepted_count": 0,
"edited_count": 0,
"rejected_count": 0,
"missed_count": 0,
"failed_count": 0,
"adoption_rate": 0,
"missed_trigger_count": 0,
"wrong_trigger_count": 0,
"bad_output_count": 0,
"script_error_count": 0,
"missing_resource_count": 0,
"review_overdue_count": 0,
"risk_band": "low",
"event_types": {
"review_event": 1
},
"failure_types": {},
"source_types": {
"manual": 1
},
"command_counts": {}
},
"next_iteration_candidates": [],
"privacy_contract": {
"storage": "local-first",
"event_scope": "metadata-only",
"raw_content_allowed": false,
"raw_event_log_packaged": false,
"blocked_fields": [
"content",
"input",
"inputs",
"message",
"messages",
"note",
"output",
"outputs",
"prompt",
"raw",
"text",
"transcript"
]
},
"failures": []
},
"review_waivers": {
"ok": true,
"summary": {
"waiver_count": 0,
"active_count": 0,
"expired_count": 0,
"invalid_count": 0,
"covered_gate_count": 0,
"covered_gate_keys": [],
"waiver_candidate_count": 2,
"waiverable_open_count": 1,
"non_waivable_count": 1
},
"policy": {
"blocker_waivers_allowed": false,
"minimum_reason_chars": 20,
"expires_required": true,
"known_gate_keys": [
"context-budget",
"intent-canvas",
"operations-loop",
"output-lab",
"permission-gates",
"permission-runtime",
"registry-audit",
"release-notes",
"runtime-matrix",
"skill-atlas",
"trigger-lab",
"trust-report"
]
},
"failures": [],
"warnings": []
},
"review_annotations": {
"ok": true,
"summary": {
"annotation_count": 0,
"open_count": 0,
"resolved_count": 0,
"deferred_count": 0,
"open_blocker_count": 0,
"open_warning_count": 0,
"linked_gate_count": 0,
"target_missing_count": 0,
"failure_count": 0
},
"annotations": [],
"failures": []
},
"world_class_evidence_plan": {
"ok": true,
"summary": {
"audit_decision": "continue-iteration",
"world_class_ready": false,
"audit_world_class_ready": false,
"ready_to_claim_world_class": false,
"ledger_completion_required": true,
"evidence_requirement_count": 4,
"task_count": 4,
"human_task_count": 1,
"external_task_count": 3,
"review_task_count": 0,
"decision": "collect-external-evidence"
},
"tasks": [
{
"key": "provider-holdout",
"label": "Provider Holdout",
"status": "external_required",
"category": "external",
"owner": "operator with provider credentials",
"current": "model-executed 0; token-observed 0",
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
"runbook": [
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"success_checks": [
"reports/output_execution_runs.json summary.model_executed_count > 0",
"reports/output_execution_runs.json summary.timing_observed_count > 0",
"reports/output_execution_runs.json summary.token_observed_count > 0",
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
],
"evidence_artifacts": [
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/skill_os2_audit.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/provider-holdout.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not commit provider credentials or environment dumps.",
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
],
"audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence."
},
{
"key": "human-adjudication",
"label": "Human Adjudication",
"status": "human_required",
"category": "human",
"owner": "human reviewer",
"current": "0/5 decisions; pending 5",
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
"runbook": [
"python3 scripts/yao.py output-review-kit --write-template",
"Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.",
"python3 scripts/adjudicate_output_review.py --write-template",
"Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.",
"python3 scripts/yao.py output-review",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"success_checks": [
"reports/output_review_adjudication.json summary.pending_count == 0",
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
],
"evidence_artifacts": [
"reports/output_blind_review_pack.md",
"reports/output_review_kit.md",
"reports/output_review_decisions.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/human-adjudication.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Reviewer decisions should not include raw user data or private customer detail.",
"Keep the answer key separate until after decisions are recorded."
],
"audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication."
},
{
"key": "native-permission-enforcement",
"label": "Native Permission Enforcement",
"status": "external_required",
"category": "external",
"owner": "target client or installer integrator",
"current": "native-enforced targets 0; installer-enforced targets 4",
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
"runbook": [
"Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.",
"Update the generated target adapter only when the guard is actually enforced by that target.",
"python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip",
"python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation",
"python3 scripts/yao.py runtime-permissions . --package-dir dist",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
"dist/targets/*/adapter.json",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"reports/install_simulation.json",
"reports/install_simulation.md",
"security/permission_policy.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-permission-enforcement.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not mark native_enforcement true for metadata-only fallbacks.",
"Keep residual risks visible for targets that still rely on operator enforcement."
],
"audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement."
},
{
"key": "native-client-telemetry",
"label": "Native Client Telemetry",
"status": "external_required",
"category": "external",
"owner": "Browser/Chrome/IDE/provider client integrator",
"current": "external source events 0; adoption samples 0",
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
"runbook": [
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
"Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.",
"python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl",
"python3 scripts/yao.py skill-atlas --workspace-root .",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"success_checks": [
"reports/adoption_drift_report.json summary.source_types.external > 0",
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
],
"evidence_artifacts": [
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.md",
"reports/telemetry_hook_recipes.json",
"scripts/telemetry_native_host.py",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-client-telemetry.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Telemetry must remain metadata-only and local-first.",
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
],
"audit_next_action": "Install a real client against the native host and import production metadata-only events."
}
],
"source_audit": {
"json": "reports/skill_os2_audit.json",
"markdown": "reports/skill_os2_audit.md",
"open_gap_count": 4
}
},
"world_class_evidence_ledger": {
"ok": true,
"summary": {
"ledger_entry_count": 4,
"source_accepted_count": 0,
"accepted_count": 0,
"pending_count": 4,
"human_pending_count": 1,
"external_pending_count": 3,
"submitted_entry_count": 0,
"missing_submission_count": 4,
"invalid_submission_count": 0,
"source_check_count": 13,
"source_pass_count": 6,
"source_blocked_count": 7,
"submitted_but_pending_count": 0,
"source_accepted_without_valid_submission_count": 0,
"overclaim_guard_active": true,
"ready_to_claim_world_class": false,
"decision": "evidence-pending"
},
"entries": [
{
"key": "provider-holdout",
"label": "Provider Holdout",
"category": "external",
"owner": "operator with provider credentials",
"status": "pending",
"source_status": "external_required",
"source_accepted": false,
"current": "model-executed 0; token-observed 0",
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
"runbook": [
"YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"provenance_requirements": [
"provider-backed model run",
"observed timing",
"observed token metadata"
],
"success_checks": [
"reports/output_execution_runs.json summary.model_executed_count > 0",
"reports/output_execution_runs.json summary.timing_observed_count > 0",
"reports/output_execution_runs.json summary.token_observed_count > 0",
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
],
"evidence_artifacts": [
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/skill_os2_audit.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/provider-holdout.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not commit provider credentials or environment dumps.",
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
],
"observed_state": {
"model_executed_count": 0,
"timing_observed_count": 10,
"token_observed_count": 0,
"accepted": false
},
"source_checklist": [
{
"evidence_key": "provider-holdout",
"label": "Provider model run",
"field": "model_executed_count",
"expected": ">0",
"actual": 0,
"status": "blocked",
"source_accepted": false,
"next_action": "Run provider-backed output-exec with real credentials."
},
{
"evidence_key": "provider-holdout",
"label": "Timing observed",
"field": "timing_observed_count",
"expected": ">0",
"actual": 10,
"status": "pass",
"source_accepted": false,
"next_action": "Provider execution should record timing metadata."
},
{
"evidence_key": "provider-holdout",
"label": "Token usage observed",
"field": "token_observed_count",
"expected": ">0",
"actual": 0,
"status": "blocked",
"source_accepted": false,
"next_action": "Provider execution should return non-estimated token usage."
}
],
"source_check_count": 3,
"source_pass_count": 1,
"source_blocked_count": 2,
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/provider-holdout.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence."
},
{
"key": "human-adjudication",
"label": "Human Adjudication",
"category": "human",
"owner": "human reviewer",
"status": "pending",
"source_status": "human_required",
"source_accepted": false,
"current": "0/5 decisions; pending 5",
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
"runbook": [
"python3 scripts/yao.py output-review-kit --write-template",
"Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.",
"python3 scripts/adjudicate_output_review.py --write-template",
"Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.",
"python3 scripts/yao.py output-review",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"provenance_requirements": [
"real reviewer identity",
"blind A/B decisions",
"answer key unopened until decisions exist"
],
"success_checks": [
"reports/output_review_adjudication.json summary.pending_count == 0",
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
],
"evidence_artifacts": [
"reports/output_blind_review_pack.md",
"reports/output_review_kit.md",
"reports/output_review_decisions.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/human-adjudication.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Reviewer decisions should not include raw user data or private customer detail.",
"Keep the answer key separate until after decisions are recorded."
],
"observed_state": {
"pair_count": 5,
"judgment_count": 0,
"pending_count": 5,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"accepted": false
},
"source_checklist": [
{
"evidence_key": "human-adjudication",
"label": "Review pairs exist",
"field": "pair_count",
"expected": ">0",
"actual": 5,
"status": "pass",
"source_accepted": false,
"next_action": "Generate the blind A/B review pack."
},
{
"evidence_key": "human-adjudication",
"label": "No pending decisions",
"field": "pending_count",
"expected": "==0",
"actual": 5,
"status": "blocked",
"source_accepted": false,
"next_action": "Record a reviewer choice for every pair."
},
{
"evidence_key": "human-adjudication",
"label": "Judgments complete",
"field": "judgment_count",
"expected": "==pair_count",
"actual": 0,
"status": "blocked",
"source_accepted": false,
"next_action": "Every pair needs one valid human judgment."
},
{
"evidence_key": "human-adjudication",
"label": "No invalid decisions",
"field": "invalid_decision_count",
"expected": "==0",
"actual": 0,
"status": "pass",
"source_accepted": false,
"next_action": "Fix malformed winner/confidence entries."
}
],
"source_check_count": 4,
"source_pass_count": 2,
"source_blocked_count": 2,
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/human-adjudication.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Record real A/B choices in the decision template, then regenerate adjudication."
},
{
"key": "native-permission-enforcement",
"label": "Native Permission Enforcement",
"category": "external",
"owner": "target client or installer integrator",
"status": "pending",
"source_status": "external_required",
"source_accepted": false,
"current": "native-enforced targets 0; installer-enforced targets 4",
"objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.",
"runbook": [
"Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.",
"Update the generated target adapter only when the guard is actually enforced by that target.",
"python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip",
"python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation",
"python3 scripts/yao.py runtime-permissions . --package-dir dist",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"provenance_requirements": [
"real target client or external installer runtime guard",
"native enforcement flag or externally accepted guard proof",
"residual risk retained for fallback targets"
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
"dist/targets/*/adapter.json",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"reports/install_simulation.json",
"reports/install_simulation.md",
"security/permission_policy.json",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-permission-enforcement.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Do not mark native_enforcement true for metadata-only fallbacks.",
"Keep residual risks visible for targets that still rely on operator enforcement."
],
"observed_state": {
"native_enforcement_count": 0,
"metadata_fallback_count": 4,
"installer_enforcement_pass_count": 4,
"installer_permission_failure_count": 0,
"installer_enforcement_ready": true,
"residual_risk_count": 4,
"failure_count": 0,
"accepted": false
},
"source_checklist": [
{
"evidence_key": "native-permission-enforcement",
"label": "Native enforcement",
"field": "native_enforcement_count",
"expected": ">0",
"actual": 0,
"status": "blocked",
"source_accepted": false,
"next_action": "Collect real target-client or external runtime guard proof."
},
{
"evidence_key": "native-permission-enforcement",
"label": "Probe failures",
"field": "failure_count",
"expected": "==0",
"actual": 0,
"status": "pass",
"source_accepted": false,
"next_action": "Runtime permission probes must stay clean."
},
{
"evidence_key": "native-permission-enforcement",
"label": "Installer support",
"field": "installer_enforcement_ready",
"expected": "true",
"actual": true,
"status": "pass",
"source_accepted": false,
"next_action": "Installer enforcement is supporting evidence, not native proof."
}
],
"source_check_count": 3,
"source_pass_count": 2,
"source_blocked_count": 1,
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/native-permission-enforcement.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement."
},
{
"key": "native-client-telemetry",
"label": "Native Client Telemetry",
"category": "external",
"owner": "Browser/Chrome/IDE/provider client integrator",
"status": "pending",
"source_status": "external_required",
"source_accepted": false,
"current": "external source events 0; adoption samples 0",
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
"runbook": [
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
"Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.",
"python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl",
"python3 scripts/yao.py skill-atlas --workspace-root .",
"python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>",
"Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.",
"python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
],
"provenance_requirements": [
"real external client source",
"metadata-only event",
"local-first import path"
],
"success_checks": [
"reports/adoption_drift_report.json summary.source_types.external > 0",
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
],
"evidence_artifacts": [
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.md",
"reports/telemetry_hook_recipes.json",
"scripts/telemetry_native_host.py",
"evidence/world_class/intake.schema.json",
"evidence/world_class/templates/native-client-telemetry.intake.json",
"reports/world_class_evidence_intake.json",
"reports/world_class_evidence_intake.md"
],
"privacy_contract": [
"Telemetry must remain metadata-only and local-first.",
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
],
"observed_state": {
"external_source_events": 0,
"adoption_sample_count": 0,
"raw_content_allowed": false,
"risk_band": "low",
"accepted": false
},
"source_checklist": [
{
"evidence_key": "native-client-telemetry",
"label": "External events",
"field": "external_source_events",
"expected": ">0",
"actual": 0,
"status": "blocked",
"source_accepted": false,
"next_action": "Import at least one metadata-only event from a real client."
},
{
"evidence_key": "native-client-telemetry",
"label": "Adoption sample",
"field": "adoption_sample_count",
"expected": ">0",
"actual": 0,
"status": "blocked",
"source_accepted": false,
"next_action": "Telemetry must include adoption outcome evidence."
},
{
"evidence_key": "native-client-telemetry",
"label": "Raw content blocked",
"field": "raw_content_allowed",
"expected": "false",
"actual": false,
"status": "pass",
"source_accepted": false,
"next_action": "Telemetry must stay metadata-only."
}
],
"source_check_count": 3,
"source_pass_count": 1,
"source_blocked_count": 2,
"submission_state": {
"status": "missing",
"path": "evidence/world_class/submissions/native-client-telemetry.json",
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Install a real client against the native host and import production metadata-only events."
}
],
"source_plan": {
"json": "reports/world_class_evidence_plan.json",
"markdown": "reports/world_class_evidence_plan.md",
"task_count": 4,
"evidence_requirement_count": 4
}
},
"synthesis_highlights": [
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed.",
"Borrow the discipline of defining what the skill should not own before growing the package."
],
"artifact_design": {
"design_system": "metric editorial",
"highlights": [
"Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
"Name the working directory and required inputs for commands.",
"Mark destructive, networked, or external side-effect operations.",
"Prefer the smallest runnable snippet over broad framework scaffolding."
]
},
"prompt_quality": {
"overall_quality_score": 89.0,
"highlights": [
"Primary prompt task family: Execution operation.",
"Complexity: expert — multiple task families plus governance, evaluation, or expert-level constraints",
"Completeness: 100/100.",
"Clarity: 85/100."
]
},
"system_model": {
"stability": {
"score": 100,
"band": "system-ready"
},
"highlights": [
"Stability: system-ready (100/100).",
"Owned job: Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"Leverage: Tune the frontmatter description — Name the recurring job, expected input, output, and strongest non-goal in compact language.",
"Leverage: Install output self-repair checks — Add only the checks that prevent recurring output mistakes."
]
},
"metadata": {
"canonical_format": "agent-skills",
"targets": [
"openai",
"claude",
"generic",
"vscode"
],
"maturity_tier": "governed",
"skill_archetype": "governed",
"updated_at": "2026-03-31"
}
},
"intent_confidence": {
"score": 100,
"band": "high",
"gate_passed": true,
"strengths": [
"The recurring job is concrete enough to anchor the package.",
"Real input shape is explicit.",
"The hand-back output is concrete.",
"Boundary exclusions are already explicit.",
"Operational constraints are visible."
],
"gaps": [],
"follow_up_questions": [],
"anchor_sentence": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse. Primary output: A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.. Exclusions: one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill, general code review or debugging unless the user is packaging that workflow as a skill, raw private material that was not intentionally supplied as skill evidence, platform-specific plugin creation when the request is not about an agent skill package.",
"recommended_action": "Intent is clear enough to package the first routeable version.",
"context": {
"job": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"real_inputs": [
"rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions",
"an existing skill directory that needs refactor, evaluation, packaging, or governance hardening",
"target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution",
"benchmark references, local constraints, desired maturity tier, and review standards"
],
"primary_output": "A working skill package with lean SKILL.md, aligned agents/interface.yaml, justified references, scripts only when useful, eval evidence, reports, packaging metadata, and clear next iteration recommendations.",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"exclusions": [
"one-off writing, translation, explanation, or brainstorming requests that do not need a reusable skill",
"general code review or debugging unless the user is packaging that workflow as a skill",
"raw private material that was not intentionally supplied as skill evidence",
"platform-specific plugin creation when the request is not about an agent skill package"
],
"constraints": [
"keep SKILL.md lean and route primarily through frontmatter description",
"put durable guidance in references, executable logic in scripts, and evidence in reports",
"default to the lightest reliable mode before adding governance weight",
"preserve portability across OpenAI, Claude, generic, and Agent Skills compatible targets",
"avoid raw prompt, output, transcript, or private content in telemetry"
],
"standards": [
"trigger boundaries must be tested with should-trigger and should-not-trigger cases",
"production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence",
"governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation",
"generated reports should be bilingual or reviewer-friendly when they are user-facing",
"each new asset must earn its place by reducing ambiguity, risk, or repeated work"
],
"correction": "The 2.0 direction reframes this package from a Meta Skill factory into a Skill OS with Skill IR, compiler contracts, output eval, runtime conformance, trust, registry, Review Studio, Skill Atlas, and telemetry loops.",
"user_references": [
"Yao Meta Skill 2.0 upgrade plan",
"Kami-style white report layout requirements for generated skill reports",
"local Waza engineering habits for review, design, diagnosis, and implementation discipline"
]
}
},
"intent": {
"skill_name": "yao-meta-skill",
"title": "Yao Meta Skill",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"focus": "portability-and-contract",
"opening_frame": "Let's start from the real work, the result you care about, and the standards that matter here. We can make the structure clearer after that.",
"reference_note": "If you already have examples you admire, bring them in. We will learn the pattern, not copy the source.",
"conversation_path": [
"Start with the user's own words, not package vocabulary.",
"Reflect the job, output, and non-goals back in one clean sentence.",
"Only then offer a tiny scaffold if it would help the user move faster."
],
"opening_styles": [
{
"label": "温柔陪伴型",
"best_when": "用户想法还散、还在试探，或者需要先被接住。",
"message": "我们先不急着把它说成一个很完整的 skill。你就像跟我聊天一样，先说说你最想让它以后稳稳接住哪类重复工作；如果它做得很理想，最后应该交回你一个什么结果。"
},
{
"label": "专业教练型",
"best_when": "用户目标比较明确，希望被高效带着走。",
"message": "我们先把这件事讲清楚，再决定 skill 怎么设计。你先告诉我三件事：它要接住的重复任务是什么，别人通常会给它什么材料，最后你希望它交付什么结果。"
},
{
"label": "共创伙伴型",
"best_when": "用户已经有一些想法，希望一起打磨，不想被填表。",
"message": "我们把这次当成一次共创。你先给我一个粗糙版本就行，我先帮你看它真正的核心任务是什么，再一起决定边界、结构和接下来最值的一步。"
}
],
"optional_scaffold": [
"The repeated job it should reliably handle",
"The real inputs people will hand to it",
"The useful output it should hand back",
"What it should clearly refuse"
],
"questions": [
{
"question": "If this skill worked beautifully, what recurring job would it reliably handle for the user every time?",
"why": "This reveals the real job-to-be-done and gives the package a humane center instead of a guessed prompt shape."
},
{
"question": "When someone reaches for this skill in the real world, what materials will they actually hand to it?",
"why": "Input shape decides whether references, scripts, or templates are needed."
},
{
"question": "What finished output should it hand back so the user can immediately keep moving?",
"why": "Outputs should drive the package structure before extra guidance is added."
},
{
"question": "Which nearby requests should this skill politely refuse so the boundary stays clean?",
"why": "The exclusion list is the fastest route to better trigger quality."
},
{
"question": "What matters most here: speed, consistency, auditability, portability, governance, or tone/style fit?",
"why": "Constraints decide how much structure, packaging, and review the skill actually needs."
},
{
"question": "Do you already have any references you want this skill to learn from, such as a repo, product, page, workflow, or prompt example?",
"why": "A good reference can raise the quality bar quickly, but the skill should only borrow patterns and standards, never copy wording or confidential material."
},
{
"question": "Which environments or clients must be able to consume this skill?",
"why": "This sets the minimum metadata and degradation strategy."
}
],
"output": {
"capability_sentence": "Yao Meta Skill should turn a recurring request into a reliable reusable output without widening the boundary unnecessarily.",
"required_capture": [
"recurring job",
"real inputs",
"required outputs",
"exclusions",
"constraints",
"reference preferences",
"first evaluation target"
],
"recommended_first_gate": "portability and contract"
}
},
"reference": {
"skill_name": "yao-meta-skill",
"title": "Yao Meta Skill",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"scan_focus": [
{
"label": "Evaluation pattern",
"reason": "This skill already carries eval assets, so benchmark how top examples define trigger boundaries and quality gates."
},
{
"label": "Execution pattern",
"reason": "There is deterministic logic in scripts, so compare how strong references separate prose from executable steps."
},
{
"label": "Portability pattern",
"reason": "The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source."
},
{
"label": "Method pattern",
"reason": "Use the core job description as the anchor for comparison: Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse."
}
],
"external_references": [],
"user_references": [],
"local_constraints": [],
"borrow_plan": [
"External benchmark first: let high-quality public references define the upper bound for method, structure, execution, or portability.",
"User references second: use them to understand taste, standards, and directional preferences without copying source phrasing.",
"Local fit third: use local assets only to detect naming conflicts, private dependencies, or compatibility constraints.",
"Borrow patterns, not prose: extract loops, boundaries, metadata, and operator flow without copying source-specific language.",
"Keep the package light: reject any borrowed pattern that increases context cost faster than it increases reliability."
],
"priority_rule": "External benchmark objects set the pattern ceiling. User references refine taste and standards. Local files only calibrate fit, risk, and compatibility.",
"non_goals": [
"Do not copy source prose or branding into the new skill.",
"Do not import gates that cost more context than they save.",
"Do not use benchmark scanning to justify scope creep.",
"Do not let local historical habits outrank stronger public benchmarks."
]
},
"iteration": {
"summary": {
"skill_name": "yao-meta-skill",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"maturity_tier": "governed",
"selection_rule": "Pick the three smallest next steps that increase reliability more than they increase context cost.",
"recommended_now": "Borrow one proven pattern on purpose",
"recommended_now_why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.",
"defer_for_now": "Create an iteration evidence loop"
},
"directions": [
{
"priority": 2,
"title": "Borrow one proven pattern on purpose",
"why": "You already have public benchmark objects. The next gain is to choose one pattern intentionally instead of absorbing everything loosely.",
"actions": [
"Read the strongest pattern from obra/superpowers.",
"Decide whether to borrow method, structure, execution, or portability, but only one of them first.",
"Record what you will not borrow so the package stays light."
],
"unlocks": "A cleaner package shape with less accidental over-design.",
"do_now": "Do this first.",
"wait_on": "Wait to add broader structure until this move clearly improves reliability."
},
{
"priority": 5,
"title": "Harden portability semantics",
"why": "The skill already signals reuse across environments, so contract clarity matters early.",
"actions": [
"Confirm activation mode, execution context, and trust assumptions.",
"Add or review degradation strategy for non-native targets.",
"Package the skill once to verify adapter expectations."
],
"unlocks": "Safer cross-environment reuse with less target drift.",
"do_now": "Do this after the first move lands cleanly.",
"wait_on": "Wait until the package has evidence that this extra structure is justified."
},
{
"priority": 12,
"title": "Create an iteration evidence loop",
"why": "The package should show what changed and why after the first draft.",
"actions": [
"Generate the HTML skill report and keep it aligned with the package.",
"Record reference scan choices and non-goals.",
"Capture the next iteration choice explicitly before adding more files."
],
"unlocks": "A clearer path for the next author or reviewer.",
"do_now": "Do this after the first move lands cleanly.",
"wait_on": "Wait until the package has evidence that this extra structure is justified."
}
]
},
"feedback": {},
"baseline": {
"summary": {
"target_count": 3,
"baseline_total_errors": 8,
"current_total_errors": 5,
"winner_total_errors": 5,
"winner_vs_baseline_gain": 3,
"winner_vs_current_gain": 0
},
"comparisons": [
{
"label": "root",
"winner_label": "Current",
"baseline": {
"tokens": 8,
"dev_errors": 1,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"current": {
"tokens": 65,
"dev_errors": 0,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"winner": {
"tokens": 65,
"dev_errors": 0,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"delta": {
"current_vs_baseline": 1,
"winner_vs_baseline": 1,
"winner_vs_current": 0
}
},
{
"label": "team-frontend-review",
"winner_label": "Current",
"baseline": {
"tokens": 52,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"current": {
"tokens": 50,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"winner": {
"tokens": 50,
"dev_errors": 3,
"holdout_errors": 0,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"delta": {
"current_vs_baseline": 0,
"winner_vs_baseline": 0,
"winner_vs_current": 0
}
},
{
"label": "governed-incident-command",
"winner_label": "Current",
"baseline": {
"tokens": 93,
"dev_errors": 1,
"holdout_errors": 2,
"blind_errors": 1,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"current": {
"tokens": 37,
"dev_errors": 1,
"holdout_errors": 1,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"winner": {
"tokens": 37,
"dev_errors": 1,
"holdout_errors": 1,
"blind_errors": 0,
"judge_blind_errors": 0,
"adversarial_errors": 0
},
"delta": {
"current_vs_baseline": 2,
"winner_vs_baseline": 2,
"winner_vs_current": 0
}
}
]
},
"compare": {
"current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"current_candidate": {
"id": "current",
"label": "Current",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"strategy": "current",
"estimated_tokens": 65,
"dev": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
0,
-1.0,
-1.0,
-1.0,
-1.0,
65
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
"baseline": {
"description": "Create and improve agent skills.",
"estimated_tokens": 8,
"dev": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.875,
"bucket_stats": {
"should_trigger": {
"total": 8,
"passed": 7,
"pass_rate": 0.875
},
"should_not_trigger": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
}
},
"family_stats": {
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 2,
"passed": 1,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.5
},
"complex_multi_asset": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_not_package": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"partial_scaffold_not_full_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "iterate_existing_skill",
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"score": 0.328,
"reason": "false_negative",
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill"
],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill",
"score": 0.387,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"checklist",
"operations checklist"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill",
"score": 0.328,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.302,
"support_score": 0.04,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"trigger evals"
],
"iterate_existing_skill": [
"this skill",
"sharing it with the team",
"before sharing it with the team"
],
"team_operationalize": [
"team",
"sharing it with the team"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset",
"score": 0.844,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0,
"lexical_support": 0.042,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package",
"turn all of that into one reusable skill package"
],
"eval_optimize": [
"evals",
"packaging checks"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"runbook",
"process",
"prompt history"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset",
"score": 0.408,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0.06,
"lexical_support": 0.067,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"production-ready"
],
"transform_workflow": [
"workflow",
"workflow fragments"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill",
"score": 0.527,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.491,
"support_score": 0.04,
"lexical_support": 0.071,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"distribution"
],
"team_operationalize": [
"team",
"distribution"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
"family": "paraphrase_trigger",
"score": 0.403,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0.04,
"lexical_support": 0.071,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"team library",
"maintained"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
"family": "paraphrase_trigger",
"score": 0.856,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0.04,
"lexical_support": 0.067,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this workflow"
],
"eval_optimize": [
"validate",
"route boundary"
],
"reuse_package": [
"package",
"distribution",
"shared capability"
],
"team_operationalize": [
"shared",
"distribution"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
"family": "long_context_trigger",
"score": 0.869,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset",
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0.1,
"lexical_support": 0.038,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill",
"skill package"
],
"eval_optimize": [
"validation"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reuse",
"package",
"team reuse"
],
"team_operationalize": [
"team",
"team reuse"
],
"transform_workflow": [
"rough notes",
"transcript",
"onboarding flow"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Give me ideas for improving our process.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"process"
]
},
"negative": {
"brainstorm_only": [
"ideas"
]
}
}
}
},
{
"prompt": "Review this note and tell me what it means in plain English.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"tell me what it means",
"plain english"
]
}
}
}
},
{
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0.06,
"lexical_support": 0.067,
"negative_penalty": 0.66,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm",
"improve packaging"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"translate_only"
],
"exclusive_negative_concepts": [
"translate_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.35,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"translate_only": [
"translate",
"into french"
]
}
}
}
},
{
"prompt": "Summarize this workflow and list the main points only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"summarize_only": [
"summarize"
]
}
}
}
},
{
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.043,
"negative_penalty": 0.68,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
]
},
"negative": {
"explain_only": [
"explain it",
"in simpler language"
],
"no_build_directive": [
"do not convert it into a reusable capability",
"keep it as documentation only"
]
}
}
}
},
{
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
"family": "long_context_summary_only",
"score": 0.257,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.698,
"support_score": 0,
"lexical_support": 0.056,
"negative_penalty": 0.3,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"process"
]
},
"negative": {
"summarize_only": [
"summary",
"recap"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.1,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"explain_only": [
"explain it"
],
"no_build_directive": [
"no packaging needed"
]
}
}
}
},
{
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.071,
"negative_penalty": 0.42,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"checklist"
]
},
"negative": {
"no_build_directive": [
"stop short of making a full skill"
]
}
}
}
},
{
"prompt": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.1,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document"
],
"no_build_directive": [
"not as an agent skill"
]
}
}
}
},
{
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.34,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"iterate_existing_skill": [
"this skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.071,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"reusable"
]
},
"negative": {
"document_only": [
"document",
"reusable-looking document"
],
"no_build_directive": [
"without turning them into an agent skill"
]
}
}
}
},
{
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.048,
"negative_penalty": 0.82,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document",
"template only"
],
"future_outline": [
"not now",
"later but not now"
],
"no_build_directive": [
"not now",
"later but not now"
]
}
}
}
},
{
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.053,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"for later",
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
}
]
}
},
"holdout": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 5,
"passed": 5,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
}
},
"family_stats": {
"complex_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset",
"score": 0.416,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset",
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0.1,
"lexical_support": 0.053,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable"
],
"team_operationalize": [
"internal"
],
"transform_workflow": [
"process",
"internal process note"
]
},
"negative": {}
}
}
},
{
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill",
"score": 0.699,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.679,
"support_score": 0.04,
"lexical_support": 0.056,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"near-neighbor",
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"package",
"distribution"
],
"team_operationalize": [
"distribution"
],
"transform_workflow": [
"existing skill draft"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill",
"score": 0.546,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.528,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"validation"
],
"reuse_package": [
"reusable",
"library entry"
],
"transform_workflow": [
"process"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
"family": "paraphrase_trigger",
"score": 0.861,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0.06,
"lexical_support": 0.062,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"packaging checks"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"quarterly release routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
"family": "long_context_trigger",
"score": 0.719,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.698,
"support_score": 0.04,
"lexical_support": 0.08,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"turn this playbook into an agent capability package"
],
"reuse_package": [
"package",
"team library"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"prompt history",
"transcript",
"release notes"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"process"
]
},
"negative": {
"no_build_directive": [
"not a reusable skill"
],
"summarize_only": [
"summary",
"only want a summary"
]
}
}
}
},
{
"prompt": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"package"
],
"transform_workflow": [
"checklist"
]
},
"negative": {
"explain_only": [
"explain this",
"in simpler language"
],
"no_build_directive": [
"do not package it"
]
}
}
}
},
{
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document"
],
"no_build_directive": [
"not for agent execution"
]
}
}
}
},
{
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"translate_only"
],
"semantic_coverage": 0.698,
"support_score": 0,
"lexical_support": 0.062,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"no_build_directive": [
"no skill package",
"keep it as documentation only"
],
"translate_only": [
"translate",
"into russian"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"semantic_coverage": 0.321,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.46,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
]
},
"negative": {
"brainstorm_only": [
"shape an idea"
],
"future_outline": [
"before we decide whether to build"
]
}
}
}
},
{
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.071,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill",
"outline for a possible future skill"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.077,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document",
"document package",
"wiki"
],
"no_build_directive": [
"not an agent skill"
]
}
}
}
},
{
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.698,
"support_score": 0,
"lexical_support": 0.125,
"negative_penalty": 0.6,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"workflow",
"checklist"
]
},
"negative": {
"document_only": [
"template only"
],
"no_build_directive": [
"no skill package",
"discussion only"
]
}
}
}
}
]
}
}
},
"winner": {
"id": "current",
"label": "Current",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"strategy": "current",
"estimated_tokens": 65,
"dev": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
0,
-1.0,
-1.0,
-1.0,
-1.0,
65
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
"winner_dev_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 8,
"passed": 8,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
}
},
"family_stats": {
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"complex_multi_asset": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_not_package": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"partial_scaffold_not_full_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill",
"score": 0.415,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"checklist",
"operations checklist"
]
},
"negative": {}
}
}
},
{
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill",
"score": 0.394,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.417,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"trigger evals"
],
"iterate_existing_skill": [
"this skill",
"sharing it with the team",
"before sharing it with the team"
],
"team_operationalize": [
"team",
"sharing it with the team"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset",
"score": 0.843,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package",
"turn all of that into one reusable skill package"
],
"eval_optimize": [
"evals",
"packaging checks"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"runbook",
"process",
"prompt history"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset",
"score": 0.422,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"production-ready"
],
"transform_workflow": [
"workflow",
"workflow fragments"
]
},
"negative": {}
}
}
},
{
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill",
"score": 0.557,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"distribution"
],
"team_operationalize": [
"team",
"distribution"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
"family": "paraphrase_trigger",
"score": 0.474,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0,
"lexical_support": 0.571,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"reuse_package": [
"team library",
"maintained"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
"family": "paraphrase_trigger",
"score": 0.868,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0,
"lexical_support": 0.267,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this workflow"
],
"eval_optimize": [
"validate",
"route boundary"
],
"reuse_package": [
"package",
"distribution",
"shared capability"
],
"team_operationalize": [
"shared",
"distribution"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
"family": "long_context_trigger",
"score": 0.889,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0.06,
"lexical_support": 0.346,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill",
"skill package"
],
"eval_optimize": [
"validation"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reuse",
"package",
"team reuse"
],
"team_operationalize": [
"team",
"team reuse"
],
"transform_workflow": [
"rough notes",
"transcript",
"onboarding flow"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Give me ideas for improving our process.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"process"
]
},
"negative": {
"brainstorm_only": [
"ideas"
]
}
}
}
},
{
"prompt": "Review this note and tell me what it means in plain English.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"tell me what it means",
"plain english"
]
}
}
}
},
{
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.2,
"negative_penalty": 0.66,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm",
"improve packaging"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"translate_only"
],
"exclusive_negative_concepts": [
"translate_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.35,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"translate_only": [
"translate",
"into french"
]
}
}
}
},
{
"prompt": "Summarize this workflow and list the main points only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"summarize_only": [
"summarize"
]
}
}
}
},
{
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.174,
"negative_penalty": 0.68,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
]
},
"negative": {
"explain_only": [
"explain it",
"in simpler language"
],
"no_build_directive": [
"do not convert it into a reusable capability",
"keep it as documentation only"
]
}
}
}
},
{
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
"family": "long_context_summary_only",
"score": 0.256,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.333,
"negative_penalty": 0.3,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"process"
]
},
"negative": {
"summarize_only": [
"summary",
"recap"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"explain_only": [
"explain it"
],
"no_build_directive": [
"no packaging needed"
]
}
}
}
},
{
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0.42,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"checklist"
]
},
"negative": {
"no_build_directive": [
"stop short of making a full skill"
]
}
}
}
},
{
"prompt": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document"
],
"no_build_directive": [
"not as an agent skill"
]
}
}
}
},
{
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.267,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"iterate_existing_skill": [
"this skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"reusable"
]
},
"negative": {
"document_only": [
"document",
"reusable-looking document"
],
"no_build_directive": [
"without turning them into an agent skill"
]
}
}
}
},
{
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.238,
"negative_penalty": 0.82,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document",
"template only"
],
"future_outline": [
"not now",
"later but not now"
],
"no_build_directive": [
"not now",
"later but not now"
]
}
}
}
},
{
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.263,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"for later",
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
}
]
}
},
"winner_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 5,
"passed": 5,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
}
},
"family_stats": {
"complex_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset",
"score": 0.468,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0.06,
"lexical_support": 0.316,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable"
],
"team_operationalize": [
"internal"
],
"transform_workflow": [
"process",
"internal process note"
]
},
"negative": {}
}
}
},
{
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill",
"score": 0.727,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.691,
"support_score": 0,
"lexical_support": 0.389,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"near-neighbor",
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"package",
"distribution"
],
"team_operationalize": [
"distribution"
],
"transform_workflow": [
"existing skill draft"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill",
"score": 0.568,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"validation"
],
"reuse_package": [
"reusable",
"library entry"
],
"transform_workflow": [
"process"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
"family": "paraphrase_trigger",
"score": 0.858,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0.06,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"packaging checks"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"quarterly release routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
"family": "long_context_trigger",
"score": 0.744,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.709,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"turn this playbook into an agent capability package"
],
"reuse_package": [
"package",
"team library"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"prompt history",
"transcript",
"release notes"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"process"
]
},
"negative": {
"no_build_directive": [
"not a reusable skill"
],
"summarize_only": [
"summary",
"only want a summary"
]
}
}
}
},
{
"prompt": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"package"
],
"transform_workflow": [
"checklist"
]
},
"negative": {
"explain_only": [
"explain this",
"in simpler language"
],
"no_build_directive": [
"do not package it"
]
}
}
}
},
{
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.222,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document"
],
"no_build_directive": [
"not for agent execution"
]
}
}
}
},
{
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"translate_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"no_build_directive": [
"no skill package",
"keep it as documentation only"
],
"translate_only": [
"translate",
"into russian"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"semantic_coverage": 0.309,
"support_score": 0,
"lexical_support": 0.308,
"negative_penalty": 0.46,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
]
},
"negative": {
"brainstorm_only": [
"shape an idea"
],
"future_outline": [
"before we decide whether to build"
]
}
}
}
},
{
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill",
"outline for a possible future skill"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.615,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document",
"document package",
"wiki"
],
"no_build_directive": [
"not an agent skill"
]
}
}
}
},
{
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.438,
"negative_penalty": 0.6,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"workflow",
"checklist"
]
},
"negative": {
"document_only": [
"template only"
],
"no_build_directive": [
"no skill package",
"discussion only"
]
}
}
}
}
]
}
},
"current_dev_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 8,
"passed": 8,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
}
},
"family_stats": {
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"complex_multi_asset": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_not_package": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"partial_scaffold_not_full_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill",
"score": 0.415,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"checklist",
"operations checklist"
]
},
"negative": {}
}
}
},
{
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill",
"score": 0.394,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.417,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"trigger evals"
],
"iterate_existing_skill": [
"this skill",
"sharing it with the team",
"before sharing it with the team"
],
"team_operationalize": [
"team",
"sharing it with the team"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset",
"score": 0.843,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package",
"turn all of that into one reusable skill package"
],
"eval_optimize": [
"evals",
"packaging checks"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"runbook",
"process",
"prompt history"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset",
"score": 0.422,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"production-ready"
],
"transform_workflow": [
"workflow",
"workflow fragments"
]
},
"negative": {}
}
}
},
{
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill",
"score": 0.557,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"distribution"
],
"team_operationalize": [
"team",
"distribution"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
"family": "paraphrase_trigger",
"score": 0.474,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0,
"lexical_support": 0.571,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"reuse_package": [
"team library",
"maintained"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
"family": "paraphrase_trigger",
"score": 0.868,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0,
"lexical_support": 0.267,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this workflow"
],
"eval_optimize": [
"validate",
"route boundary"
],
"reuse_package": [
"package",
"distribution",
"shared capability"
],
"team_operationalize": [
"shared",
"distribution"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
"family": "long_context_trigger",
"score": 0.889,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0.06,
"lexical_support": 0.346,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill",
"skill package"
],
"eval_optimize": [
"validation"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reuse",
"package",
"team reuse"
],
"team_operationalize": [
"team",
"team reuse"
],
"transform_workflow": [
"rough notes",
"transcript",
"onboarding flow"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Give me ideas for improving our process.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"process"
]
},
"negative": {
"brainstorm_only": [
"ideas"
]
}
}
}
},
{
"prompt": "Review this note and tell me what it means in plain English.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"tell me what it means",
"plain english"
]
}
}
}
},
{
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.2,
"negative_penalty": 0.66,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm",
"improve packaging"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"translate_only"
],
"exclusive_negative_concepts": [
"translate_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.35,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"translate_only": [
"translate",
"into french"
]
}
}
}
},
{
"prompt": "Summarize this workflow and list the main points only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"summarize_only": [
"summarize"
]
}
}
}
},
{
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.174,
"negative_penalty": 0.68,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
]
},
"negative": {
"explain_only": [
"explain it",
"in simpler language"
],
"no_build_directive": [
"do not convert it into a reusable capability",
"keep it as documentation only"
]
}
}
}
},
{
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
"family": "long_context_summary_only",
"score": 0.256,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.333,
"negative_penalty": 0.3,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"process"
]
},
"negative": {
"summarize_only": [
"summary",
"recap"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"explain_only": [
"explain it"
],
"no_build_directive": [
"no packaging needed"
]
}
}
}
},
{
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0.42,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"checklist"
]
},
"negative": {
"no_build_directive": [
"stop short of making a full skill"
]
}
}
}
},
{
"prompt": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document"
],
"no_build_directive": [
"not as an agent skill"
]
}
}
}
},
{
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.267,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"iterate_existing_skill": [
"this skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"reusable"
]
},
"negative": {
"document_only": [
"document",
"reusable-looking document"
],
"no_build_directive": [
"without turning them into an agent skill"
]
}
}
}
},
{
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.238,
"negative_penalty": 0.82,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document",
"template only"
],
"future_outline": [
"not now",
"later but not now"
],
"no_build_directive": [
"not now",
"later but not now"
]
}
}
}
},
{
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.263,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"for later",
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
}
]
}
},
"current_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 5,
"passed": 5,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
}
},
"family_stats": {
"complex_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset",
"score": 0.468,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0.06,
"lexical_support": 0.316,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable"
],
"team_operationalize": [
"internal"
],
"transform_workflow": [
"process",
"internal process note"
]
},
"negative": {}
}
}
},
{
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill",
"score": 0.727,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.691,
"support_score": 0,
"lexical_support": 0.389,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"near-neighbor",
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"package",
"distribution"
],
"team_operationalize": [
"distribution"
],
"transform_workflow": [
"existing skill draft"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill",
"score": 0.568,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"validation"
],
"reuse_package": [
"reusable",
"library entry"
],
"transform_workflow": [
"process"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
"family": "paraphrase_trigger",
"score": 0.858,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0.06,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"packaging checks"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"quarterly release routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
"family": "long_context_trigger",
"score": 0.744,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.709,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"turn this playbook into an agent capability package"
],
"reuse_package": [
"package",
"team library"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"prompt history",
"transcript",
"release notes"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"process"
]
},
"negative": {
"no_build_directive": [
"not a reusable skill"
],
"summarize_only": [
"summary",
"only want a summary"
]
}
}
}
},
{
"prompt": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"package"
],
"transform_workflow": [
"checklist"
]
},
"negative": {
"explain_only": [
"explain this",
"in simpler language"
],
"no_build_directive": [
"do not package it"
]
}
}
}
},
{
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.222,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document"
],
"no_build_directive": [
"not for agent execution"
]
}
}
}
},
{
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"translate_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"no_build_directive": [
"no skill package",
"keep it as documentation only"
],
"translate_only": [
"translate",
"into russian"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"semantic_coverage": 0.309,
"support_score": 0,
"lexical_support": 0.308,
"negative_penalty": 0.46,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
]
},
"negative": {
"brainstorm_only": [
"shape an idea"
],
"future_outline": [
"before we decide whether to build"
]
}
}
}
},
{
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill",
"outline for a possible future skill"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.615,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document",
"document package",
"wiki"
],
"no_build_directive": [
"not an agent skill"
]
}
}
}
},
{
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.438,
"negative_penalty": 0.6,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"workflow",
"checklist"
]
},
"negative": {
"document_only": [
"template only"
],
"no_build_directive": [
"no skill package",
"discussion only"
]
}
}
}
}
]
}
},
"winner_blind_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"score": 0.392,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.214,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"score": 0.405,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"score": 0.028,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.318,
"negative_penalty": 0.18,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
}
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.3,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.0,
"support_score": 0.06,
"lexical_support": 0.308,
"negative_penalty": 0.22,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
}
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"score": 0.026,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.273,
"negative_penalty": 0.22,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
}
}
]
}
},
"current_blind_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"score": 0.392,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.214,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"score": 0.405,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"score": 0.028,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.318,
"negative_penalty": 0.18,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
}
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.3,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.0,
"support_score": 0.06,
"lexical_support": 0.308,
"negative_penalty": 0.22,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
}
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"score": 0.026,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.273,
"negative_penalty": 0.22,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
}
}
]
}
},
"baseline_blind_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"score": 0.393,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.071,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"score": 0.392,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.062,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"score": 0.019,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.189,
"support_score": 0.06,
"lexical_support": 0.136,
"negative_penalty": 0.18,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
}
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.0,
"support_score": 0.06,
"lexical_support": 0.0,
"negative_penalty": 0.22,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
}
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"score": 0.024,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.22,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
}
}
]
}
},
"winner_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 0,
"margin": 3,
"confidence": 0.74,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow"
],
"strong_positive_concepts": [
"transform_workflow"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 3,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [
"document_only"
],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 2,
"negative_vote": 1,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"prompts"
],
"artifact_hits": [
"prompts"
],
"capability_overlap": 2,
"exclusive_negative_concepts": [
"explain_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain",
"plain english"
],
"positive_vote": 2,
"negative_vote": 3,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 2,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 2,
"margin": 1,
"confidence": 0.66,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
},
"boundary_case": true
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"current_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 0,
"margin": 3,
"confidence": 0.74,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow"
],
"strong_positive_concepts": [
"transform_workflow"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 3,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [
"document_only"
],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 2,
"negative_vote": 1,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"prompts"
],
"artifact_hits": [
"prompts"
],
"capability_overlap": 2,
"exclusive_negative_concepts": [
"explain_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain",
"plain english"
],
"positive_vote": 2,
"negative_vote": 3,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 2,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 2,
"margin": 1,
"confidence": 0.66,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
},
"boundary_case": true
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"baseline_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 0,
"margin": 3,
"confidence": 0.74,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow"
],
"strong_positive_concepts": [
"transform_workflow"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 3,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [
"document_only"
],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 2,
"negative_vote": 1,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"prompts"
],
"artifact_hits": [
"prompts"
],
"capability_overlap": 2,
"exclusive_negative_concepts": [
"explain_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain",
"plain english"
],
"positive_vote": 2,
"negative_vote": 3,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 2,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 2,
"margin": 1,
"confidence": 0.66,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
},
"boundary_case": true
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"winner_adversarial_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_noisy_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_multi_asset_boundary": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_document_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_summary_translation_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_outline_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_brainstorm_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
"family": "adversarial_noisy_operationalize",
"score": 0.834,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.263,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"hardening"
],
"reuse_package": [
"package",
"maintained",
"operationalize"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
"family": "adversarial_multi_asset_boundary",
"score": 0.842,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.368,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this routine"
],
"eval_optimize": [
"stress test",
"route boundary"
],
"reuse_package": [
"reusable"
],
"transform_workflow": [
"prompt history"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
"family": "adversarial_document_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.429,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"document_only": [
"wiki"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
"family": "adversarial_summary_translation_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive",
"summarize_only",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only",
"translate_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.188,
"negative_penalty": 1.25,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"sop"
]
},
"negative": {
"document_only": [
"knowledge base"
],
"no_build_directive": [
"keep it as documentation only"
],
"summarize_only": [
"summarize"
],
"translate_only": [
"translate",
"into japanese"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
"family": "adversarial_future_outline_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill"
],
"no_build_directive": [
"stop short of making a full skill",
"plain note",
"keep it as a plain note"
]
}
}
}
},
{
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
"family": "adversarial_brainstorm_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.182,
"negative_penalty": 0.88,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
],
"future_outline": [
"for later"
],
"no_build_directive": [
"discussion only"
]
}
}
}
}
]
}
},
"current_adversarial_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_noisy_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_multi_asset_boundary": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_document_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_summary_translation_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_outline_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_brainstorm_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
"family": "adversarial_noisy_operationalize",
"score": 0.834,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.263,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"hardening"
],
"reuse_package": [
"package",
"maintained",
"operationalize"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
"family": "adversarial_multi_asset_boundary",
"score": 0.842,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.368,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this routine"
],
"eval_optimize": [
"stress test",
"route boundary"
],
"reuse_package": [
"reusable"
],
"transform_workflow": [
"prompt history"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
"family": "adversarial_document_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.429,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"document_only": [
"wiki"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
"family": "adversarial_summary_translation_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive",
"summarize_only",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only",
"translate_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.188,
"negative_penalty": 1.25,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"sop"
]
},
"negative": {
"document_only": [
"knowledge base"
],
"no_build_directive": [
"keep it as documentation only"
],
"summarize_only": [
"summarize"
],
"translate_only": [
"translate",
"into japanese"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
"family": "adversarial_future_outline_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill"
],
"no_build_directive": [
"stop short of making a full skill",
"plain note",
"keep it as a plain note"
]
}
}
}
},
{
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
"family": "adversarial_brainstorm_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.182,
"negative_penalty": 0.88,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
],
"future_outline": [
"for later"
],
"no_build_directive": [
"discussion only"
]
}
}
}
}
]
}
},
"baseline_adversarial_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_noisy_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_multi_asset_boundary": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_document_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_summary_translation_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_outline_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_brainstorm_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
"family": "adversarial_noisy_operationalize",
"score": 0.845,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0,
"lexical_support": 0.053,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"hardening"
],
"reuse_package": [
"package",
"maintained",
"operationalize"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
"family": "adversarial_multi_asset_boundary",
"score": 0.845,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0,
"lexical_support": 0.053,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this routine"
],
"eval_optimize": [
"stress test",
"route boundary"
],
"reuse_package": [
"reusable"
],
"transform_workflow": [
"prompt history"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
"family": "adversarial_document_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.377,
"support_score": 0.06,
"lexical_support": 0.143,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"document_only": [
"wiki"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
"family": "adversarial_summary_translation_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive",
"summarize_only",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only",
"translate_only"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.062,
"negative_penalty": 1.25,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"sop"
]
},
"negative": {
"document_only": [
"knowledge base"
],
"no_build_directive": [
"keep it as documentation only"
],
"summarize_only": [
"summarize"
],
"translate_only": [
"translate",
"into japanese"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
"family": "adversarial_future_outline_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.05,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill"
],
"no_build_directive": [
"stop short of making a full skill",
"plain note",
"keep it as a plain note"
]
}
}
}
},
{
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
"family": "adversarial_brainstorm_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.88,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
],
"future_outline": [
"for later"
],
"no_build_directive": [
"discussion only"
]
}
}
}
}
]
}
},
"candidates": [
{
"id": "current",
"label": "Current",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"strategy": "current",
"estimated_tokens": 65,
"dev": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
0,
-1.0,
-1.0,
-1.0,
-1.0,
65
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "minimal",
"label": "Minimal",
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"strategy": "minimal_template",
"estimated_tokens": 41,
"dev": {
"false_positives": 1,
"false_negatives": 1,
"precision": 0.875,
"recall": 0.875,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
1,
1,
-1.0,
-0.857,
-0.875,
-0.875,
41
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "guardrail",
"label": "Guardrail",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"strategy": "guardrail_template",
"estimated_tokens": 56,
"dev": {
"false_positives": 1,
"false_negatives": 1,
"precision": 0.875,
"recall": 0.875,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
1,
1,
-1.0,
-0.857,
-0.875,
-0.875,
56
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "balanced",
"label": "Balanced",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"strategy": "balanced_template",
"estimated_tokens": 60,
"dev": {
"false_positives": 1,
"false_negatives": 1,
"precision": 0.875,
"recall": 0.875,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
1,
1,
-1.0,
-0.857,
-0.875,
-0.875,
60
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "artifact_aware",
"label": "Artifact Aware",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"strategy": "artifact_template",
"estimated_tokens": 77,
"dev": {
"false_positives": 1,
"false_negatives": 1,
"precision": 0.875,
"recall": 0.875,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
1,
1,
-1.0,
-0.857,
-0.875,
-0.875,
77
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "boundary",
"label": "Boundary",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"strategy": "boundary_template",
"estimated_tokens": 83,
"dev": {
"false_positives": 1,
"false_negatives": 1,
"precision": 0.875,
"recall": 0.875,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
1,
1,
-1.0,
-0.857,
-0.875,
-0.875,
83
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
}
],
"selection_logic": {
"priority": [
"fewest false positives",
"fewest false negatives",
"highest near-neighbor pass rate",
"highest negative pass rate",
"highest precision",
"highest recall",
"shortest description"
]
},
"comparison": {
"winner_vs_current_dev": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_dev": {
"baseline_false_positives": 0,
"baseline_false_negatives": 1,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": -1,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.875,
"improved_recall": 1.0
},
"winner_vs_baseline_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_judge_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_judge_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_adversarial_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_adversarial_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
}
},
"acceptance_gates": {
"selection_basis": "dev only",
"holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.673,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
"max_non_trigger_score": 0.0,
"score_gap": 0.468,
"threshold_margin": 0.138,
"positive_threshold_buffer": 0.138,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"current_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.673,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
"max_non_trigger_score": 0.0,
"score_gap": 0.468,
"threshold_margin": 0.138,
"positive_threshold_buffer": 0.138,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"baseline_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.648,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.416,
"max_non_trigger_score": 0.0,
"score_gap": 0.416,
"threshold_margin": 0.086,
"positive_threshold_buffer": 0.086,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"winner_family_health": {
"family_count": 12,
"clean_family_count": 12,
"failing_families": [],
"weakest_family": {
"family": "workflow_to_skill",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 12,
"clean_family_count": 12,
"failing_families": [],
"weakest_family": {
"family": "workflow_to_skill",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 12,
"clean_family_count": 12,
"failing_families": [],
"weakest_family": {
"family": "workflow_to_skill",
"pass_rate": 1.0,
"errors": 0
}
}
},
"blind_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.399,
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.028,
"score_gap": 0.364,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.302,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
},
"current_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.399,
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.028,
"score_gap": 0.364,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.302,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
},
"baseline_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.393,
"mean_non_trigger_score": 0.011,
"mean_near_neighbor_score": 0.012,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.024,
"score_gap": 0.368,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.306,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
},
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
}
},
"judge_blind_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"winner_calibration": null,
"current_calibration": null,
"baseline_calibration": null,
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
}
},
"adversarial_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.838,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.834,
"max_non_trigger_score": 0.0,
"score_gap": 0.834,
"threshold_margin": 0.33,
"positive_threshold_buffer": 0.504,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"current_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.838,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.834,
"max_non_trigger_score": 0.0,
"score_gap": 0.834,
"threshold_margin": 0.33,
"positive_threshold_buffer": 0.504,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"baseline_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.845,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.845,
"max_non_trigger_score": 0.0,
"score_gap": 0.845,
"threshold_margin": 0.33,
"positive_threshold_buffer": 0.515,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_summary_translation_collision",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_summary_translation_collision",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_summary_translation_collision",
"pass_rate": 1.0,
"errors": 0
}
}
}
},
"summary": {
"winner_label": "Current",
"winner_tokens": 65,
"current_tokens": 65,
"winner_dev_total_errors": 0,
"current_dev_total_errors": 0,
"winner_holdout_total_errors": 0,
"current_holdout_total_errors": 0,
"winner_blind_holdout_total_errors": 0,
"current_blind_holdout_total_errors": 0,
"winner_judge_blind_holdout_total_errors": 0,
"current_judge_blind_holdout_total_errors": 0,
"winner_adversarial_holdout_total_errors": 0,
"current_adversarial_holdout_total_errors": 0,
"winner_judge_blind_agreement_rate": 1.0,
"winner_adversarial_risk_band": "healthy",
"winner_adversarial_score_gap": 0.834,
"candidate_count": 6,
"baseline_tokens": 8,
"baseline_dev_total_errors": 1,
"baseline_holdout_total_errors": 0,
"baseline_blind_holdout_total_errors": 0,
"baseline_judge_blind_holdout_total_errors": 0,
"baseline_adversarial_holdout_total_errors": 0
}
},
"promotion": {
"ok": true,
"summary": {
"promote": 0,
"keep_current": 3,
"blocked": 0
},
"targets": [
{
"name": "yao-meta-skill",
"winner_label": "Current",
"winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"winner_tokens": 65,
"current_tokens": 65,
"winner_holdout_fp": 0,
"winner_holdout_fn": 0,
"current_holdout_fp": 0,
"current_holdout_fn": 0,
"baseline_holdout_fp": 0,
"baseline_holdout_fn": 0,
"winner_blind_holdout_fp": 0,
"winner_blind_holdout_fn": 0,
"current_blind_holdout_fp": 0,
"current_blind_holdout_fn": 0,
"baseline_blind_holdout_fp": 0,
"baseline_blind_holdout_fn": 0,
"winner_blind_holdout_total_errors": 0,
"winner_judge_blind_holdout_fp": 0,
"winner_judge_blind_holdout_fn": 0,
"current_judge_blind_holdout_fp": 0,
"current_judge_blind_holdout_fn": 0,
"baseline_judge_blind_holdout_fp": 0,
"baseline_judge_blind_holdout_fn": 0,
"winner_judge_blind_holdout_total_errors": 0,
"winner_adversarial_holdout_fp": 0,
"winner_adversarial_holdout_fn": 0,
"current_adversarial_holdout_fp": 0,
"current_adversarial_holdout_fn": 0,
"baseline_adversarial_holdout_fp": 0,
"baseline_adversarial_holdout_fn": 0,
"winner_adversarial_holdout_total_errors": 0,
"calibration": {
"holdout": {
"threshold": 0.33,
"mean_positive_score": 0.673,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
"max_non_trigger_score": 0.0,
"score_gap": 0.468,
"threshold_margin": 0.138,
"positive_threshold_buffer": 0.138,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"blind_holdout": {
"threshold": 0.33,
"mean_positive_score": 0.399,
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.028,
"score_gap": 0.364,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.302,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
},
"adversarial_holdout": {
"threshold": 0.33,
"mean_positive_score": 0.838,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.834,
"max_non_trigger_score": 0.0,
"score_gap": 0.834,
"threshold_margin": 0.33,
"positive_threshold_buffer": 0.504,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
}
},
"judge_blind": {
"winner": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
},
"current": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
},
"baseline": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"family_health": {
"holdout": {
"family_count": 12,
"clean_family_count": 12,
"failing_families": [],
"weakest_family": {
"family": "workflow_to_skill",
"pass_rate": 1.0,
"errors": 0
}
},
"blind_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"judge_blind_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"adversarial_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_summary_translation_collision",
"pass_rate": 1.0,
"errors": 0
}
}
},
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
"ok": true,
"promotion": {
"decision": "keep_current",
"winner_changed": false,
"all_gates_pass": true,
"route_clean": true,
"judge_agreement_ok": true,
"causes": [
"no_candidate_outperformed_current"
],
"next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.",
"gate_status": {
"visible_holdout_non_regression": true,
"blind_holdout_non_regression": true,
"judge_blind_non_regression": true,
"judge_blind_agreement": true,
"adversarial_non_regression": true,
"adversarial_risk_ok": true,
"route_confusion_clean": true,
"family_stability": true
},
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.611,
"misroute_count": 0,
"ambiguous_case_count": 0
}
},
"registry": [
{
"name": "yao-meta-skill",
"role": "baseline",
"label": "Baseline",
"ranking_state": "reference",
"promotion_state": "reference",
"description": "Create and improve agent skills.",
"tokens": 8,
"dev_errors": 1,
"holdout_errors": 0
},
{
"name": "yao-meta-skill",
"role": "current",
"label": "Current",
"ranking_state": "selected_by_dev",
"promotion_state": "kept_current",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"tokens": 65,
"dev_errors": 0,
"holdout_errors": 0,
"reason_tags": []
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Minimal",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 41,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 56,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Balanced",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 60,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Artifact Aware",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"tokens": 77,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
"longer_without_gain"
]
},
{
"name": "yao-meta-skill",
"role": "candidate",
"label": "Boundary",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"tokens": 83,
"dev_errors": 2,
"holdout_errors": 0,
"reason_tags": [
"weaker_dev_fit",
"longer_without_gain"
]
}
],
"decision": "keep_current"
},
{
"name": "team-frontend-review",
"winner_label": "Current",
"winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
"winner_tokens": 50,
"current_tokens": 50,
"winner_holdout_fp": 0,
"winner_holdout_fn": 0,
"current_holdout_fp": 0,
"current_holdout_fn": 0,
"baseline_holdout_fp": 0,
"baseline_holdout_fn": 0,
"winner_blind_holdout_fp": 0,
"winner_blind_holdout_fn": 0,
"current_blind_holdout_fp": 0,
"current_blind_holdout_fn": 0,
"baseline_blind_holdout_fp": 0,
"baseline_blind_holdout_fn": 0,
"winner_blind_holdout_total_errors": 0,
"winner_judge_blind_holdout_fp": 0,
"winner_judge_blind_holdout_fn": 0,
"current_judge_blind_holdout_fp": 0,
"current_judge_blind_holdout_fn": 0,
"baseline_judge_blind_holdout_fp": 0,
"baseline_judge_blind_holdout_fn": 0,
"winner_judge_blind_holdout_total_errors": 0,
"winner_adversarial_holdout_fp": 0,
"winner_adversarial_holdout_fn": 0,
"current_adversarial_holdout_fp": 0,
"current_adversarial_holdout_fn": 0,
"baseline_adversarial_holdout_fp": 0,
"baseline_adversarial_holdout_fn": 0,
"winner_adversarial_holdout_total_errors": 0,
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.089,
"mean_near_neighbor_score": 0.045,
"min_positive_score": 0.496,
"max_non_trigger_score": 0.211,
"score_gap": 0.285,
"threshold_margin": 0.016,
"positive_threshold_buffer": 0.016,
"negative_threshold_buffer": 0.269,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
"risk_band": "tight"
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.685,
"mean_non_trigger_score": 0.177,
"mean_near_neighbor_score": 0.226,
"min_positive_score": 0.504,
"max_non_trigger_score": 0.416,
"score_gap": 0.088,
"threshold_margin": 0.024,
"positive_threshold_buffer": 0.024,
"negative_threshold_buffer": 0.064,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "tight"
},
"adversarial_holdout": {
"threshold": 0.34,
"mean_positive_score": 0.71,
"mean_non_trigger_score": 0.009,
"mean_near_neighbor_score": 0.018,
"min_positive_score": 0.544,
"max_non_trigger_score": 0.035,
"score_gap": 0.509,
"threshold_margin": 0.204,
"positive_threshold_buffer": 0.204,
"negative_threshold_buffer": 0.305,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
}
},
"judge_blind": {
"winner": {
"agreement_rate": 1.0,
"mean_confidence": 0.703,
"rubric_version": "blind-v1"
},
"current": {
"agreement_rate": 1.0,
"mean_confidence": 0.703,
"rubric_version": "blind-v1"
},
"baseline": {
"agreement_rate": 1.0,
"mean_confidence": 0.703,
"rubric_version": "blind-v1"
}
},
"family_health": {
"holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "security_and_a11y",
"pass_rate": 1.0,
"errors": 0
}
},
"blind_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_premerge",
"pass_rate": 1.0,
"errors": 0
}
},
"judge_blind_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_premerge",
"pass_rate": 1.0,
"errors": 0
}
},
"adversarial_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_quality_gate_review",
"pass_rate": 1.0,
"errors": 0
}
}
},
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
"ok": true,
"promotion": {
"decision": "keep_current",
"winner_changed": false,
"all_gates_pass": true,
"route_clean": true,
"judge_agreement_ok": true,
"causes": [
"no_candidate_outperformed_current",
"current_holdout_risk"
],
"next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.",
"gate_status": {
"visible_holdout_non_regression": true,
"blind_holdout_non_regression": true,
"judge_blind_non_regression": true,
"judge_blind_agreement": true,
"adversarial_non_regression": true,
"adversarial_risk_ok": true,
"route_confusion_clean": true,
"family_stability": true
},
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.805,
"misroute_count": 0,
"ambiguous_case_count": 0
}
},
"registry": [
{
"name": "team-frontend-review",
"role": "baseline",
"label": "Baseline",
"ranking_state": "reference",
"promotion_state": "reference",
"description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.",
"tokens": 52,
"dev_errors": 3,
"holdout_errors": 0
},
{
"name": "team-frontend-review",
"role": "current",
"label": "Current",
"ranking_state": "selected_by_dev",
"promotion_state": "kept_current",
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
"tokens": 50,
"dev_errors": 3,
"holdout_errors": 0,
"reason_tags": []
},
{
"name": "team-frontend-review",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.",
"tokens": 62,
"dev_errors": 3,
"holdout_errors": 0,
"reason_tags": [
"longer_without_gain"
]
},
{
"name": "team-frontend-review",
"role": "candidate",
"label": "Balanced",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
"tokens": 64,
"dev_errors": 3,
"holdout_errors": 0,
"reason_tags": [
"longer_without_gain"
]
},
{
"name": "team-frontend-review",
"role": "candidate",
"label": "Artifact Aware",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Trigger when requests mention React components, forms, frontend diffs, or UI code and the job is to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
"tokens": 84,
"dev_errors": 3,
"holdout_errors": 0,
"reason_tags": [
"longer_without_gain"
]
},
{
"name": "team-frontend-review",
"role": "candidate",
"label": "Boundary",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.",
"tokens": 90,
"dev_errors": 3,
"holdout_errors": 0,
"reason_tags": [
"longer_without_gain"
]
}
],
"decision": "keep_current"
},
{
"name": "governed-incident-command",
"winner_label": "Current",
"winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"winner_tokens": 37,
"current_tokens": 37,
"winner_holdout_fp": 0,
"winner_holdout_fn": 1,
"current_holdout_fp": 0,
"current_holdout_fn": 1,
"baseline_holdout_fp": 0,
"baseline_holdout_fn": 2,
"winner_blind_holdout_fp": 0,
"winner_blind_holdout_fn": 0,
"current_blind_holdout_fp": 0,
"current_blind_holdout_fn": 0,
"baseline_blind_holdout_fp": 0,
"baseline_blind_holdout_fn": 1,
"winner_blind_holdout_total_errors": 0,
"winner_judge_blind_holdout_fp": 0,
"winner_judge_blind_holdout_fn": 0,
"current_judge_blind_holdout_fp": 0,
"current_judge_blind_holdout_fn": 0,
"baseline_judge_blind_holdout_fp": 0,
"baseline_judge_blind_holdout_fn": 0,
"winner_judge_blind_holdout_total_errors": 0,
"winner_adversarial_holdout_fp": 0,
"winner_adversarial_holdout_fn": 0,
"current_adversarial_holdout_fp": 0,
"current_adversarial_holdout_fn": 0,
"baseline_adversarial_holdout_fp": 0,
"baseline_adversarial_holdout_fn": 0,
"winner_adversarial_holdout_total_errors": 0,
"calibration": {
"holdout": {
"threshold": 0.48,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"blind_holdout": {
"threshold": 0.48,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
"risk_band": "healthy"
},
"adversarial_holdout": {
"threshold": 0.34,
"mean_positive_score": 0.827,
"mean_non_trigger_score": 0.027,
"mean_near_neighbor_score": 0.011,
"min_positive_score": 0.654,
"max_non_trigger_score": 0.056,
"score_gap": 0.598,
"threshold_margin": 0.284,
"positive_threshold_buffer": 0.314,
"negative_threshold_buffer": 0.284,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
}
},
"judge_blind": {
"winner": {
"agreement_rate": 1.0,
"mean_confidence": 0.657,
"rubric_version": "blind-v1"
},
"current": {
"agreement_rate": 1.0,
"mean_confidence": 0.657,
"rubric_version": "blind-v1"
},
"baseline": {
"agreement_rate": 1.0,
"mean_confidence": 0.69,
"rubric_version": "blind-v1"
}
},
"family_health": {
"holdout": {
"family_count": 6,
"clean_family_count": 5,
"failing_families": [
{
"family": "packet_assembly",
"errors": 1,
"pass_rate": 0.0
}
],
"weakest_family": {
"family": "packet_assembly",
"pass_rate": 0.0,
"errors": 1
}
},
"blind_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_summary_only",
"pass_rate": 1.0,
"errors": 0
}
},
"judge_blind_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_summary_only",
"pass_rate": 1.0,
"errors": 0
}
},
"adversarial_holdout": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_single_update_collision",
"pass_rate": 1.0,
"errors": 0
}
}
},
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
"ok": true,
"promotion": {
"decision": "keep_current",
"winner_changed": false,
"all_gates_pass": true,
"route_clean": true,
"judge_agreement_ok": true,
"causes": [
"no_candidate_outperformed_current",
"current_holdout_gap_present",
"current_holdout_risk"
],
"next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.",
"gate_status": {
"visible_holdout_non_regression": true,
"blind_holdout_non_regression": true,
"judge_blind_non_regression": true,
"judge_blind_agreement": true,
"adversarial_non_regression": true,
"adversarial_risk_ok": true,
"route_confusion_clean": true,
"family_stability": true
},
"route_metrics": {
"precision": 1.0,
"recall": 1.0,
"average_margin": 0.659,
"misroute_count": 0,
"ambiguous_case_count": 0
}
},
"registry": [
{
"name": "governed-incident-command",
"role": "baseline",
"label": "Baseline",
"ranking_state": "reference",
"promotion_state": "reference",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.",
"tokens": 93,
"dev_errors": 1,
"holdout_errors": 2
},
{
"name": "governed-incident-command",
"role": "current",
"label": "Current",
"ranking_state": "selected_by_dev",
"promotion_state": "kept_current",
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"tokens": 37,
"dev_errors": 1,
"holdout_errors": 1,
"reason_tags": []
},
{
"name": "governed-incident-command",
"role": "candidate",
"label": "Guardrail",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
"tokens": 51,
"dev_errors": 1,
"holdout_errors": 2,
"reason_tags": [
"visible_holdout_regression",
"longer_without_gain"
]
},
{
"name": "governed-incident-command",
"role": "candidate",
"label": "Balanced",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"tokens": 54,
"dev_errors": 1,
"holdout_errors": 1,
"reason_tags": [
"longer_without_gain"
]
},
{
"name": "governed-incident-command",
"role": "candidate",
"label": "Boundary",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
"tokens": 78,
"dev_errors": 1,
"holdout_errors": 1,
"reason_tags": [
"longer_without_gain"
]
},
{
"name": "governed-incident-command",
"role": "candidate",
"label": "Artifact Aware",
"ranking_state": "not_selected",
"promotion_state": "blocked",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.",
"tokens": 78,
"dev_errors": 1,
"holdout_errors": 1,
"reason_tags": [
"longer_without_gain"
]
}
],
"decision": "keep_current"
}
]
},
"benchmark": {
"ok": true,
"query": "skills",
"source": "github-api",
"network_boundary": "github-api-only",
"repositories": [
{
"full_name": "obra/superpowers",
"html_url": "https://github.com/obra/superpowers",
"description": "An agentic skills framework & software development methodology that works.",
"stars": 226125,
"topics": [],
"primary_language": "Shell",
"patterns": [
{
"category": "method",
"matched": "process",
"borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.",
"avoid": "Do not import process overhead that only exists for that project's scale."
},
{
"category": "execution",
"matched": "cli",
"borrow": "Borrow the clear execution entrypoints and command structure.",
"avoid": "Do not copy repo-specific commands or environment assumptions verbatim."
},
{
"category": "evaluation",
"matched": "test",
"borrow": "Borrow explicit validation and quality gates that make iteration safer.",
"avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here."
},
{
"category": "structure",
"matched": "docs",
"borrow": "Borrow the way it separates explanation, examples, and reusable structure.",
"avoid": "Do not mirror documentation bulk that adds context cost without improving reliability."
}
],
"borrow": [
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure.",
"Borrow explicit validation and quality gates that make iteration safer."
],
"avoid": [
"Do not import process overhead that only exists for that project's scale.",
"Do not copy repo-specific commands or environment assumptions verbatim.",
"Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here."
],
"readme_excerpt": "# Superpowers\n\nSuperpowers is a complete software development methodology for your coding agents, built on top of a set of composable skills and some initial instructions that make sure your agent uses them.\n\n## Quickstart\n\nGive your agent Superpowers: [Claude Code](#claude-code), [Codex CLI](#codex-cli), [Codex App](#codex-app), [Factory Droid](#factory-droid), [Gemini CLI](#gemini-cli), [OpenCode](#opencode), [Cursor](#cursor), [GitHub Copilot CLI](#github-copilot-cli).\n\n## How it works\n\nIt starts from the moment you fire up your coding agent. As soon as it sees that you're building something, it *doesn't* just jump into trying to write code. Instead, it steps back and asks you what you're really trying to do. \n\nOnce it's teased a spec out of the conversation, it shows it to you in chunks short enough to actually read and digest. \n\nAfter you've signed off on the design, your agent puts together an implementation plan that's clear enough for an enthusiastic junior engineer with poor taste, no judgement, no project context, and an aversion to testing to follow. It emphasizes true red/green TDD, YAGNI (You Aren't Gonna Need It), and DRY. \n\nNext up, once you say \"go\", it launches a *"
},
{
"full_name": "affaan-m/ECC",
"html_url": "https://github.com/affaan-m/ECC",
"description": "The agent harness performance optimization system. Skills, instincts, memory, security, and research-first development for Claude Code, Codex, Opencode, Cursor and beyond.",
"stars": 214381,
"topics": [
"ai-agents",
"anthropic",
"claude",
"claude-code",
"developer-tools",
"llm",
"mcp",
"productivity"
],
"primary_language": "JavaScript",
"patterns": [
{
"category": "method",
"matched": "workflow",
"borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.",
"avoid": "Do not import process overhead that only exists for that project's scale."
},
{
"category": "execution",
"matched": "command, shell, script",
"borrow": "Borrow the clear execution entrypoints and command structure.",
"avoid": "Do not copy repo-specific commands or environment assumptions verbatim."
},
{
"category": "structure",
"matched": "guide, docs",
"borrow": "Borrow the way it separates explanation, examples, and reusable structure.",
"avoid": "Do not mirror documentation bulk that adds context cost without improving reliability."
},
{
"category": "governance",
"matched": "security, review",
"borrow": "Borrow the explicit review, safety, or operational trust boundaries.",
"avoid": "Do not import compliance-heavy process where the new skill does not need it."
}
],
"borrow": [
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure.",
"Borrow the way it separates explanation, examples, and reusable structure."
],
"avoid": [
"Do not import process overhead that only exists for that project's scale.",
"Do not copy repo-specific commands or environment assumptions verbatim.",
"Do not mirror documentation bulk that adds context cost without improving reliability."
],
"readme_excerpt": "**Language:** English | [Português (Brasil)](docs/pt-BR/README.md) | [简体中文](README.zh-CN.md) | [繁體中文](docs/zh-TW/README.md) | [日本語](docs/ja-JP/README.md) | [한국어](docs/ko-KR/README.md) | [Türkçe](docs/tr/README.md) | [Русский](docs/ru/README.md) | [Tiếng Việt](docs/vi-VN/README.md) | [ไทย](docs/th/README.md) | [Deutsch](docs/de-DE/README.md) | [Español](docs/es/README.md)\n\n![ECC — the agent harness operating system](assets/hero.png)\n\n[![Stars](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fstars&style=flat)](https://github.com/affaan-m/ECC/stargazers)\n[![Forks](https://img.shields.io/endpoint?url=https%3A%2F%2Fapi.ecc.tools%2Fbadge%2Fforks&style=flat)](https://github.com/affaan-m/ECC/network/members)\n[![Contributors](https://img.shields.io/github/contributors/affaan-m/ECC?style=flat)](https://github.com/affaan-m/ECC/graphs/contributors)\n[![npm ecc-universal](https://img.shields.io/npm/dw/ecc-universal?label=ecc-universal%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-universal)\n[![npm ecc-agentshield](https://img.shields.io/npm/dw/ecc-agentshield?label=ecc-agentshield%20weekly%20downloads&logo=npm)](https://www.npmjs.com/package/ecc-agen"
},
{
"full_name": "multica-ai/andrej-karpathy-skills",
"html_url": "https://github.com/multica-ai/andrej-karpathy-skills",
"description": "A single CLAUDE.md file to improve Claude Code behavior, derived from Andrej Karpathy's observations on LLM coding pitfalls.",
"stars": 174264,
"topics": [],
"primary_language": null,
"patterns": [
{
"category": "evaluation",
"matched": "test, validation",
"borrow": "Borrow explicit validation and quality gates that make iteration safer.",
"avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here."
},
{
"category": "structure",
"matched": "guide",
"borrow": "Borrow the way it separates explanation, examples, and reusable structure.",
"avoid": "Do not mirror documentation bulk that adds context cost without improving reliability."
},
{
"category": "portability",
"matched": "plugin",
"borrow": "Borrow how it keeps core semantics stable across environments or integrations.",
"avoid": "Do not inherit platform lock-in or vendor-specific branching unless truly required."
}
],
"borrow": [
"Borrow explicit validation and quality gates that make iteration safer.",
"Borrow the way it separates explanation, examples, and reusable structure.",
"Borrow how it keeps core semantics stable across environments or integrations."
],
"avoid": [
"Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.",
"Do not mirror documentation bulk that adds context cost without improving reliability.",
"Do not inherit platform lock-in or vendor-specific branching unless truly required."
],
"readme_excerpt": "# Karpathy-Inspired Claude Code Guidelines\n\n> Check out my new project [Multica](https://github.com/multica-ai/multica) — an open-source platform for running and managing coding agents with reusable skills.\n>\n> Follow me on X: [https://x.com/jiayuan_jy](https://x.com/jiayuan_jy)\n\nA single `CLAUDE.md` file to improve Claude Code behavior, derived from [Andrej Karpathy's observations](https://x.com/karpathy/status/2015883857489522876) on LLM coding pitfalls.\n\nEnglish | [简体中文](./README.zh.md)\n\n## The Problems\n\nFrom Andrej's post:\n\n> \"The models make wrong assumptions on your behalf and just run along with them without checking. They don't manage their confusion, don't seek clarifications, don't surface inconsistencies, don't present tradeoffs, don't push back when they should.\"\n\n> \"They really like to overcomplicate code and APIs, bloat abstractions, don't clean up dead code... implement a bloated construction over 1000 lines when 100 would do.\"\n\n> \"They still sometimes change/remove comments and code they don't sufficiently understand as side effects, even if orthogonal to the task.\"\n\n## The Solution\n\nFour principles in one file that directly address these issues:\n\n| Principle | Addr"
}
],
"cross_repo": {
"borrow": [
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure.",
"Borrow explicit validation and quality gates that make iteration safer.",
"Borrow the way it separates explanation, examples, and reusable structure.",
"Borrow how it keeps core semantics stable across environments or integrations.",
"Ask the user which of these patterns feels most worth borrowing before freezing the package shape."
],
"avoid": [
"Do not import process overhead that only exists for that project's scale.",
"Do not copy repo-specific commands or environment assumptions verbatim.",
"Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.",
"Do not mirror documentation bulk that adds context cost without improving reliability.",
"Do not inherit platform lock-in or vendor-specific branching unless truly required."
]
},
"external_references": [
{
"name": "obra/superpowers",
"category": "method",
"borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.",
"avoid": "Do not import process overhead that only exists for that project's scale.",
"source": "external"
},
{
"name": "affaan-m/ECC",
"category": "method",
"borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.",
"avoid": "Do not import process overhead that only exists for that project's scale.",
"source": "external"
},
{
"name": "multica-ai/andrej-karpathy-skills",
"category": "evaluation",
"borrow": "Borrow explicit validation and quality gates that make iteration safer.",
"avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.",
"source": "external"
}
],
"borrow_prompt": "I found 3 public GitHub projects worth studying. Do you want to borrow any of these patterns for method, structure, execution, or portability?",
"warnings": []
},
"reference_synthesis": {
"skill_name": "yao-meta-skill",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"intent_confidence": {
"score": 100,
"band": "high",
"gate_passed": true
},
"github_benchmarks": [
{
"name": "obra/superpowers",
"url": "https://github.com/obra/superpowers",
"stars": 226125,
"borrow": [
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure."
]
},
{
"name": "affaan-m/ECC",
"url": "https://github.com/affaan-m/ECC",
"stars": 214381,
"borrow": [
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure."
]
},
{
"name": "multica-ai/andrej-karpathy-skills",
"url": "https://github.com/multica-ai/andrej-karpathy-skills",
"stars": 174264,
"borrow": [
"Borrow explicit validation and quality gates that make iteration safer.",
"Borrow the way it separates explanation, examples, and reusable structure."
]
}
],
"source_tracks": [
{
"source_type": "official",
"name": "Official skill anatomy and context discipline",
"evidence_mode": "curated-pattern-track",
"matched_keywords": [
"metadata",
"references"
],
"borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"avoid": "Do not let packaging or platform concerns swallow the core job boundary.",
"why_relevant": "This track matches: metadata, references."
},
{
"source_type": "research",
"name": "Human-in-the-loop verification",
"evidence_mode": "curated-pattern-track",
"matched_keywords": [
"review",
"govern"
],
"borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.",
"avoid": "Do not force every skill through heavyweight review when the risk is low.",
"why_relevant": "This track matches: review, govern."
},
{
"source_type": "principles",
"name": "Boundary-first design",
"evidence_mode": "curated-pattern-track",
"matched_keywords": [
"route"
],
"borrow": "Borrow the discipline of defining what the skill should not own before growing the package.",
"avoid": "Do not expand execution assets until route boundaries stay clean.",
"why_relevant": "This track matches: route."
}
],
"synthesis": {
"borrow_now": [
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed.",
"Borrow the discipline of defining what the skill should not own before growing the package.",
"Borrow the way it turns a messy workflow into a repeatable operating path.",
"Borrow the clear execution entrypoints and command structure."
],
"avoid_now": [
"Do not let packaging or platform concerns swallow the core job boundary.",
"Do not force every skill through heavyweight review when the risk is low.",
"Do not expand execution assets until route boundaries stay clean.",
"Do not import process overhead that only exists for that project's scale.",
"Do not copy repo-specific commands or environment assumptions verbatim."
],
"quality_risers": [
"Use GitHub repositories for concrete package and workflow patterns.",
"Use curated official or commercial tracks for entrypoint and operator ergonomics.",
"Use research tracks to justify the smallest evaluation loop that still catches regressions.",
"Use principle tracks to keep the package small, boundary-aware, and outcome-driven."
],
"pattern_gate": {
"threshold": 4,
"source_count": 6,
"accepted": [
{
"name": "Official skill anatomy and context discipline",
"source_type": "official",
"borrow": "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"avoid": "Do not let packaging or platform concerns swallow the core job boundary.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
},
{
"name": "Human-in-the-loop verification",
"source_type": "research",
"borrow": "Borrow a review checkpoint wherever trust matters more than raw speed.",
"avoid": "Do not force every skill through heavyweight review when the risk is low.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
},
{
"name": "Boundary-first design",
"source_type": "principles",
"borrow": "Borrow the discipline of defining what the skill should not own before growing the package.",
"avoid": "Do not expand execution assets until route boundaries stay clean.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
},
{
"name": "obra/superpowers",
"source_type": "github",
"borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.; Borrow the clear execution entrypoints and command structure.",
"avoid": "Do not import process overhead that only exists for that project's scale.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
},
{
"name": "affaan-m/ECC",
"source_type": "github",
"borrow": "Borrow the way it turns a messy workflow into a repeatable operating path.; Borrow the clear execution entrypoints and command structure.",
"avoid": "Do not import process overhead that only exists for that project's scale.",
"gates": {
"recurrence": true,
"generativity": true,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"generativity",
"distinctiveness",
"boundary"
],
"missing": [],
"score": 4
}
],
"deferred": [
{
"name": "multica-ai/andrej-karpathy-skills",
"source_type": "github",
"borrow": "Borrow explicit validation and quality gates that make iteration safer.; Borrow the way it separates explanation, examples, and reusable structure.",
"avoid": "Do not clone heavyweight evaluation scaffolding if a lighter gate is enough here.",
"gates": {
"recurrence": true,
"generativity": false,
"distinctiveness": true,
"boundary": true
},
"passed": [
"recurrence",
"distinctiveness",
"boundary"
],
"missing": [
"generativity"
],
"score": 3
}
],
"summary": "5 accepted, 1 deferred using threshold 4/4."
},
"conflicts": [
{
"key": "lightweight_vs_governance",
"summary": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.",
"user_preference": "lightweight or speed-first",
"benchmark_pressure": "governance or evaluation-heavy patterns"
}
],
"recommendation": {
"summary": "Start by borrowing this pattern: Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts. Avoid this for the first pass: Do not let packaging or platform concerns swallow the core job boundary.",
"borrow_now": [
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed."
],
"avoid_for_now": [
"Do not let packaging or platform concerns swallow the core job boundary.",
"Do not force every skill through heavyweight review when the risk is low."
],
"why": "There is a real design conflict to resolve: The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure.",
"user_decision_required": true
},
"visibility": {
"mode": "explicit",
"user_decision_required": true,
"reasons": [
"design_conflict"
],
"user_note": "Surface the recommendation because intent is still settling or there is a real design conflict that needs a user call.",
"reviewer_note": "Keep the full benchmark and synthesis evidence visible for authors and reviewers."
},
"decision_prompt": "Use the recommendation by default. Only surface the underlying benchmark tradeoffs when intent is uncertain or a real design conflict needs a deliberate call.",
"source_mix": {
"github_benchmarks": 3,
"curated_tracks": 3,
"user_references": 0
}
}
},
"output_risk": {
"skill_name": "yao-meta-skill",
"risk_families": [
{
"key": "markdown_readability",
"label": "Markdown readability",
"matched_keywords": [
"md",
"report",
"doc"
],
"score": 3,
"risks": [
"Tables can render as dense grids with weak hierarchy or poor mobile readability.",
"Long bullets can make the output look complete while hiding the actual decision logic.",
"Mixed heading levels can reduce scanability."
],
"constraints": [
"Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.",
"Keep table cells short and move explanations below the table.",
"Use heading levels consistently and keep each section anchored to a user-facing outcome."
],
"self_repair": [
"Preview whether each table still reads well when columns are narrow.",
"Convert any table with paragraph-length cells into bullets or cards."
]
},
{
"key": "citation_clutter",
"label": "Citation and footnote clutter",
"matched_keywords": [
"source",
"reference"
],
"score": 2,
"risks": [
"Footnote markers or dense citation notes can interrupt the reading flow.",
"Evidence can be over-attached to obvious statements and under-attached to risky claims.",
"Source notes may become more prominent than the tutorial itself."
],
"constraints": [
"Attach citations only to claims that need evidence, not to every sentence.",
"Group source notes at the end of a section when inline markers would hurt readability.",
"Keep the main sentence readable without requiring the reader to chase a footnote."
],
"self_repair": [
"Remove decorative citations that do not support a material claim.",
"Move repeated source explanations into one compact source note."
]
},
{
"key": "visual_capture",
"label": "Screenshot and visual capture",
"matched_keywords": [
"capture"
],
"score": 1,
"risks": [
"Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.",
"Missing screenshots can cause the skill to invent visual references instead of declaring the gap.",
"Image descriptions can omit the action-relevant region."
],
"constraints": [
"Never invent a screenshot; state when visual evidence is missing.",
"Record the source, viewport, and crop intent for any screenshot-dependent output.",
"Describe what the reader should inspect in the image, not just that an image exists."
],
"self_repair": [
"Check that every screenshot reference points to a real provided or generated asset.",
"Reword any visual instruction that depends on an unseen screen state."
]
},
{
"key": "code_or_command_safety",
"label": "Code and command safety",
"matched_keywords": [
"script"
],
"score": 1,
"risks": [
"Commands can omit environment assumptions, working directory, or rollback notes.",
"Code snippets can look runnable while missing required inputs.",
"Error handling can be either absent or over-engineered."
],
"constraints": [
"Name the working directory, required inputs, and expected output for each command.",
"Mark destructive or external side-effect operations explicitly.",
"Prefer the smallest runnable snippet over broad framework code."
],
"self_repair": [
"Scan each command for cwd, input, output, and side-effect assumptions.",
"Remove speculative error handling that is not tied to a real failure mode."
]
},
{
"key": "tone_and_specificity",
"label": "Tone and specificity",
"matched_keywords": [
"summary"
],
"score": 1,
"risks": [
"Headings and summaries can drift into generic, interchangeable language.",
"The output can sound polished but lose the user's actual taste, audience, or scenario.",
"Strong claims can appear without examples or constraints."
],
"constraints": [
"Anchor titles and summaries in the user's audience, object, and concrete outcome.",
"Avoid placeholder phrases such as comprehensive guide, ultimate solution, or key insights unless the source demands them.",
"Preserve one distinctive phrase, constraint, or standard from the user's input."
],
"self_repair": [
"Replace generic title candidates with scenario-specific alternatives.",
"Delete any polished sentence that could fit almost any project unchanged."
]
}
],
"top_risks": [
"Tables can render as dense grids with weak hierarchy or poor mobile readability.",
"Long bullets can make the output look complete while hiding the actual decision logic.",
"Footnote markers or dense citation notes can interrupt the reading flow.",
"Evidence can be over-attached to obvious statements and under-attached to risky claims.",
"Screenshots can be captured from the wrong state, wrong viewport, or wrong crop.",
"Missing screenshots can cause the skill to invent visual references instead of declaring the gap."
],
"output_constraints": [
"Use tables only when comparison is the main job; otherwise prefer compact cards or grouped bullets.",
"Keep table cells short and move explanations below the table.",
"Attach citations only to claims that need evidence, not to every sentence.",
"Group source notes at the end of a section when inline markers would hurt readability.",
"Never invent a screenshot; state when visual evidence is missing.",
"Record the source, viewport, and crop intent for any screenshot-dependent output."
],
"self_repair_checks": [
"Preview whether each table still reads well when columns are narrow.",
"Convert any table with paragraph-length cells into bullets or cards.",
"Remove decorative citations that do not support a material claim.",
"Move repeated source explanations into one compact source note.",
"Check that every screenshot reference points to a real provided or generated asset.",
"Reword any visual instruction that depends on an unseen screen state."
],
"reviewer_note": "Use this report before deepening the package and again before approving example outputs."
},
"artifact_design": {
"skill_name": "yao-meta-skill",
"design_system": "metric editorial",
"primary_artifact": {
"key": "code_or_cli",
"label": "Code, CLI, or implementation guide",
"direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit.",
"matched_keywords": [
"code",
"script",
"command"
]
},
"artifact_families": [
{
"key": "code_or_cli",
"label": "Code, CLI, or implementation guide",
"score": 3,
"matched_keywords": [
"code",
"script",
"command"
],
"direction": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit."
},
{
"key": "report",
"label": "Report or brief",
"score": 2,
"matched_keywords": [
"report",
"summary"
],
"direction": "High-trust editorial report with a clear first-screen thesis, compact evidence blocks, and decisions separated from supporting detail."
},
{
"key": "review_viewer",
"label": "Review viewer",
"score": 2,
"matched_keywords": [
"review",
"viewer"
],
"direction": "Side-by-side reviewer studio with explicit tradeoffs, evidence readiness, and fast paths for approving, blocking, or requesting one focused fix."
},
{
"key": "dashboard",
"label": "Dashboard or metrics page",
"score": 1,
"matched_keywords": [
"table"
],
"direction": "Metric-first dashboard with stable dimensions, short labels, visible deltas, and narrative callouts only where they change interpretation."
},
{
"key": "visual_capture",
"label": "Screenshot or visual evidence",
"score": 1,
"matched_keywords": [
"screenshot"
],
"direction": "Evidence-led visual artifact that records source, viewport, crop intent, and the exact region the reader should inspect."
}
],
"layout_patterns": [
"prerequisites",
"commands",
"expected output",
"failure handling",
"rollback or cleanup",
"thesis",
"evidence blocks",
"decision table"
],
"design_tokens": {
"type": [
"Use a distinctive display face or serif for major claims when the artifact is editorial.",
"Use a restrained sans for dense body text and technical details.",
"Use mono only for metadata, paths, commands, labels, and evidence tags."
],
"color": [
"Choose colors from the artifact's domain, brand, or evidence mood.",
"Do not default to Kami parchment, purple gradients, or generic SaaS blue unless the content justifies it.",
"Keep accent color limited to decisions, active states, risk, or section anchors."
],
"spacing": [
"Prefer clear grid rhythm over floating decorative cards.",
"Increase whitespace around decisions and shrink whitespace around supporting metadata.",
"Split dense content instead of shrinking type or adding scroll traps."
],
"components": [
"Use cards for grouped evidence, tables for comparisons, callouts for decisions, and timelines for sequence.",
"Avoid cards inside cards.",
"Keep reviewer-only detail visible but visually quieter than user-facing guidance."
]
},
"quality_gates": [
"Name the working directory and required inputs for commands.",
"Mark destructive, networked, or external side-effect operations.",
"Prefer the smallest runnable snippet over broad framework scaffolding.",
"Keep the first screen useful without requiring the reader to parse every detail.",
"Use tables only for comparisons; move explanations below the table.",
"Keep source notes readable without flooding the body with markers.",
"Make differences visible instead of hiding them in prose.",
"Separate author-facing recommendations from reviewer-only evidence."
],
"anti_patterns": [
"Do not copy Kami's fixed parchment background as a default.",
"Do not use generic purple gradients, glass cards, or stock SaaS hero sections unless the content calls for them.",
"Do not let Markdown tables become the default shape for every comparison or explanation.",
"Do not turn reviewer evidence into user-facing clutter.",
"Do not invent screenshots, citations, charts, or UI states."
],
"reviewer_note": "Use this profile to judge whether the generated artifacts feel designed for their job, not merely rendered."
},
"prompt_quality": {
"skill_name": "yao-meta-skill",
"relevance": "prompt-heavy",
"primary_task_family": {
"key": "execution_operation",
"label": "Execution operation",
"matched_keywords": [
"workflow",
"runbook"
]
},
"task_families": [
{
"key": "execution_operation",
"label": "Execution operation",
"score": 2,
"matched_keywords": [
"workflow",
"runbook"
],
"role_guidance": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.",
"task_guidance": "Convert the job into ordered steps with validation checks and stop conditions.",
"format_guidance": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant."
},
{
"key": "dialogue_interaction",
"label": "Dialogue interaction",
"score": 2,
"matched_keywords": [
"dialogue",
"conversation"
],
"role_guidance": "Use a conversational role that asks only high-leverage questions and remembers the user's goal.",
"task_guidance": "Clarify intent, resolve uncertainty, and converge toward a recommendation instead of a long option list.",
"format_guidance": "Return concise prompts, decision points, and reviewer-visible assumptions."
},
{
"key": "creative_generation",
"label": "Creative generation",
"score": 1,
"matched_keywords": [
"content"
],
"role_guidance": "Use a taste-aware creator role with clear audience, tone, and originality boundaries.",
"task_guidance": "Generate variants, explain selection logic, and preserve the user's distinctive constraints.",
"format_guidance": "Return options with rationale, selection criteria, and refinement paths."
},
{
"key": "prompt_engineering",
"label": "Prompt engineering",
"score": 1,
"matched_keywords": [
"prompt"
],
"role_guidance": "Use a prompt engineer role only when role design materially improves execution.",
"task_guidance": "Map Role, Task, and Format into skill behavior rather than copying a large prompt template.",
"format_guidance": "Return a compact prompt contract plus tests, quality matrix, and usage notes."
}
],
"complexity": {
"band": "expert",
"score": 27,
"reason": "multiple task families plus governance, evaluation, or expert-level constraints"
},
"need_model": {
"explicit_need": "Turn repeated workflows, prompts, transcripts, runbooks, documents, or existing skill packages into routeable, evaluable, packageable, and governable agent skills for personal, team, library, or governed reuse.",
"implicit_need": "The reusable skill needs a stable role, task, and output contract rather than a one-off prompt.",
"scenario": "rough workflow notes, SOPs, runbooks, prompts, transcripts, documents, or repeated task descriptions, an existing skill directory that needs refactor, evaluation, packaging, or governance hardening, target platform requirements such as OpenAI, Claude, generic Agent Skills, or team distribution, benchmark references, local constraints, desired maturity tier, and review standards",
"user_level": "infer from examples and standards; ask only if it changes output depth",
"success_standard": "trigger boundaries must be tested with should-trigger and should-not-trigger cases, production and higher maturity work needs output eval, trust, runtime conformance, and Review Studio evidence, governed work needs owner, review cadence, permission approvals, registry metadata, package verification, and install simulation, generated reports should be bilingual or reviewer-friendly when they are user-facing, each new asset must earn its place by reducing ambiguity, risk, or repeated work"
},
"rtf_to_skill": {
"role": "Use an operator role with explicit boundaries, inputs, outputs, and failure handling.",
"task": "Convert the job into ordered steps with validation checks and stop conditions.",
"format": "Return a runbook-like handoff with commands, checks, owners, and next actions when relevant."
},
"quality_matrix": [
{
"key": "completeness",
"label": "Completeness",
"score": 100,
"matched_signals": [
"output",
"constraint",
"standard"
],
"repair": "Name missing inputs, outputs, constraints, or success standards before deepening the package."
},
{
"key": "clarity",
"label": "Clarity",
"score": 85,
"matched_signals": [
"clear"
],
"repair": "Replace broad verbs with observable actions and define what done means."
},
{
"key": "consistency",
"label": "Consistency",
"score": 90,
"matched_signals": [
"aligned",
"boundary"
],
"repair": "Check that role, task, format, exclusions, and examples do not contradict each other."
},
{
"key": "practicality",
"label": "Practicality",
"score": 90,
"matched_signals": [
"use",
"workflow"
],
"repair": "Add runnable steps, examples, or verification cues instead of abstract advice."
},
{
"key": "specificity",
"label": "Specificity",
"score": 80,
"matched_signals": [],
"repair": "Anchor wording in the user's audience, domain nouns, and target outcome."
}
],
"overall_quality_score": 89.0,
"self_repair_checks": [
"Check explicit need, implicit need, scenario, user level, and success standard before deepening.",
"Map Role, Task, and Format into skill behavior, not decorative prompt labels.",
"Ask one focused clarification only when missing information changes the package boundary.",
"Add tests or examples for prompt-heavy behavior before treating it as reusable.",
"Keep prompt methodology in references and reports instead of bloating SKILL.md."
],
"reviewer_note": "Use this profile when the package depends on prompt behavior, role design, output contracts, or conversation quality."
},
"evidence_readiness": {
"score": 85,
"checks": [
{
"label": "Intent clarity",
"status": "ready",
"detail": "100/100 intent confidence."
},
{
"label": "Benchmark coverage",
"status": "ready",
"detail": "3 GitHub benchmark repositories attached."
},
{
"label": "Pattern gate",
"status": "ready",
"detail": "5 accepted, 1 deferred using threshold 4/4."
},
{
"label": "Conflict handling",
"status": "decision needed",
"detail": "The stated preference leans lightweight or speed-first, while the benchmark mix leans toward governance, review, or heavier evaluation structure."
},
{
"label": "Output risk profile",
"status": "ready",
"detail": "5 output risk families attached."
},
{
"label": "Artifact design profile",
"status": "ready",
"detail": "Execution-focused technical artifact with environment assumptions, copyable commands, expected outputs, and side effects made explicit."
},
{
"label": "Prompt quality profile",
"status": "ready",
"detail": "89.0/100 prompt-facing quality score."
}
],
"reviewer_note": "Use this section to decide whether the package is ready to deepen or should stay in discovery."
}
}
Reference in New Issue View Git Blame Copy Permalink