frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
13a67da51e2ec0097e0588e0ec51d78adbe145e9
yao-meta-skill/reports/description_optimization.json
T

10435 lines
321 KiB
JSON
Raw Blame History

{
"current_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"current_candidate": {
"id": "current",
"label": "Current",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"strategy": "current",
"estimated_tokens": 65,
"dev": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
0,
-1.0,
-1.0,
-1.0,
-1.0,
65
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
"baseline": {
"description": "Create and improve agent skills.",
"estimated_tokens": 8,
"dev": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 1,
"false_negatives": 1,
"precision": 0.875,
"recall": 0.875,
"bucket_stats": {
"should_trigger": {
"total": 8,
"passed": 7,
"pass_rate": 0.875
},
"should_not_trigger": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 7,
"passed": 6,
"pass_rate": 0.857
}
},
"family_stats": {
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 2,
"passed": 1,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.5
},
"complex_multi_asset": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_not_package": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"partial_scaffold_not_full_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 0,
"false_positives": 1,
"false_negatives": 0,
"pass_rate": 0.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "iterate_existing_skill",
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"score": 0.328,
"reason": "false_negative",
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill"
],
"matched_negative_concepts": []
},
{
"bucket": "near_neighbor",
"family": "long_context_near_neighbor",
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"score": 0.333,
"reason": "false_positive",
"matched_desired_concepts": [
"build_skill",
"transform_workflow"
],
"matched_negative_concepts": [
"document_only"
]
}
],
"results": {
"should_trigger": [
{
"prompt": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill",
"score": 0.387,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"checklist",
"operations checklist"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill",
"score": 0.328,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.302,
"support_score": 0.04,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"trigger evals"
],
"iterate_existing_skill": [
"this skill",
"sharing it with the team",
"before sharing it with the team"
],
"team_operationalize": [
"team",
"sharing it with the team"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset",
"score": 0.844,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0,
"lexical_support": 0.042,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package",
"turn all of that into one reusable skill package"
],
"eval_optimize": [
"evals",
"packaging checks"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"runbook",
"process",
"prompt history"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset",
"score": 0.408,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0.06,
"lexical_support": 0.071,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"production-ready"
],
"transform_workflow": [
"workflow",
"workflow fragments"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill",
"score": 0.527,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.491,
"support_score": 0.04,
"lexical_support": 0.071,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"distribution"
],
"team_operationalize": [
"team",
"distribution"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
"family": "paraphrase_trigger",
"score": 0.403,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0.04,
"lexical_support": 0.077,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"team library",
"maintained"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
"family": "paraphrase_trigger",
"score": 0.856,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0.04,
"lexical_support": 0.067,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this workflow"
],
"eval_optimize": [
"validate",
"route boundary"
],
"reuse_package": [
"package",
"distribution",
"shared capability"
],
"team_operationalize": [
"shared",
"distribution"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
"family": "long_context_trigger",
"score": 0.869,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset",
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0.1,
"lexical_support": 0.037,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill",
"skill package"
],
"eval_optimize": [
"validation"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reuse",
"package",
"team reuse"
],
"team_operationalize": [
"team",
"team reuse"
],
"transform_workflow": [
"rough notes",
"transcript",
"onboarding flow"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Give me ideas for improving our process.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"process"
]
},
"negative": {
"brainstorm_only": [
"ideas"
]
}
}
}
},
{
"prompt": "Review this note and tell me what it means in plain English.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"tell me what it means",
"plain english"
]
}
}
}
},
{
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0.06,
"lexical_support": 0.067,
"negative_penalty": 0.66,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm",
"improve packaging"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"translate_only"
],
"exclusive_negative_concepts": [
"translate_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.35,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"translate_only": [
"translate",
"into french"
]
}
}
}
},
{
"prompt": "Summarize this workflow and list the main points only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"summarize_only": [
"summarize"
]
}
}
}
},
{
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.045,
"negative_penalty": 0.68,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
]
},
"negative": {
"explain_only": [
"explain it",
"in simpler language"
],
"no_build_directive": [
"do not convert it into a reusable capability",
"keep it as documentation only"
]
}
}
}
},
{
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
"family": "long_context_summary_only",
"score": 0.257,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.698,
"support_score": 0,
"lexical_support": 0.059,
"negative_penalty": 0.3,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"process"
]
},
"negative": {
"summarize_only": [
"summary",
"recap"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.1,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"explain_only": [
"explain it"
],
"no_build_directive": [
"no packaging needed"
]
}
}
}
},
{
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.071,
"negative_penalty": 0.42,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"checklist"
]
},
"negative": {
"no_build_directive": [
"stop short of making a full skill"
]
}
}
}
},
{
"prompt": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.1,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document"
],
"no_build_directive": [
"not as an agent skill"
]
}
}
}
},
{
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.34,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"iterate_existing_skill": [
"this skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.077,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"reusable"
]
},
"negative": {
"document_only": [
"document",
"reusable-looking document"
],
"no_build_directive": [
"without turning them into an agent skill"
]
}
}
}
},
{
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"family": "long_context_near_neighbor",
"score": 0.333,
"predicted_trigger": true,
"expected_trigger": false,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.05,
"negative_penalty": 0.18,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document",
"template only"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.056,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"for later",
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
}
]
}
},
"holdout": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 5,
"passed": 5,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
}
},
"family_stats": {
"complex_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset",
"score": 0.416,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset",
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0.1,
"lexical_support": 0.053,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable"
],
"team_operationalize": [
"internal"
],
"transform_workflow": [
"process",
"internal process note"
]
},
"negative": {}
}
}
},
{
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill",
"score": 0.7,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.679,
"support_score": 0.04,
"lexical_support": 0.059,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"near-neighbor",
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"package",
"distribution"
],
"team_operationalize": [
"distribution"
],
"transform_workflow": [
"existing skill draft"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill",
"score": 0.546,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.528,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"validation"
],
"reuse_package": [
"reusable",
"library entry"
],
"transform_workflow": [
"process"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
"family": "paraphrase_trigger",
"score": 0.861,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0.06,
"lexical_support": 0.062,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"packaging checks"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"quarterly release routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
"family": "long_context_trigger",
"score": 0.719,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"team_operationalize"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.698,
"support_score": 0.04,
"lexical_support": 0.08,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"turn this playbook into an agent capability package"
],
"reuse_package": [
"package",
"team library"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"prompt history",
"transcript",
"release notes"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"process"
]
},
"negative": {
"no_build_directive": [
"not a reusable skill"
],
"summarize_only": [
"summary",
"only want a summary"
]
}
}
}
},
{
"prompt": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"package"
],
"transform_workflow": [
"checklist"
]
},
"negative": {
"explain_only": [
"explain this",
"in simpler language"
],
"no_build_directive": [
"do not package it"
]
}
}
}
},
{
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document"
],
"no_build_directive": [
"not for agent execution"
]
}
}
}
},
{
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"translate_only"
],
"semantic_coverage": 0.698,
"support_score": 0,
"lexical_support": 0.062,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"no_build_directive": [
"no skill package",
"keep it as documentation only"
],
"translate_only": [
"translate",
"into russian"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"semantic_coverage": 0.321,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.46,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
]
},
"negative": {
"brainstorm_only": [
"shape an idea"
],
"future_outline": [
"before we decide whether to build"
]
}
}
}
},
{
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.071,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill",
"outline for a possible future skill"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.077,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document",
"document package",
"wiki"
],
"no_build_directive": [
"not an agent skill"
]
}
}
}
},
{
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.698,
"support_score": 0,
"lexical_support": 0.062,
"negative_penalty": 0.6,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"workflow",
"checklist"
]
},
"negative": {
"document_only": [
"template only"
],
"no_build_directive": [
"no skill package",
"discussion only"
]
}
}
}
}
]
}
}
},
"winner": {
"id": "current",
"label": "Current",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"strategy": "current",
"estimated_tokens": 65,
"dev": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
0,
-1.0,
-1.0,
-1.0,
-1.0,
65
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
"winner_dev_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 8,
"passed": 8,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
}
},
"family_stats": {
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"complex_multi_asset": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_not_package": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"partial_scaffold_not_full_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill",
"score": 0.415,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"checklist",
"operations checklist"
]
},
"negative": {}
}
}
},
{
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill",
"score": 0.388,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.333,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"trigger evals"
],
"iterate_existing_skill": [
"this skill",
"sharing it with the team",
"before sharing it with the team"
],
"team_operationalize": [
"team",
"sharing it with the team"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset",
"score": 0.843,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package",
"turn all of that into one reusable skill package"
],
"eval_optimize": [
"evals",
"packaging checks"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"runbook",
"process",
"prompt history"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset",
"score": 0.418,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.357,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"production-ready"
],
"transform_workflow": [
"workflow",
"workflow fragments"
]
},
"negative": {}
}
}
},
{
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill",
"score": 0.557,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"distribution"
],
"team_operationalize": [
"team",
"distribution"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
"family": "paraphrase_trigger",
"score": 0.471,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0,
"lexical_support": 0.538,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"reuse_package": [
"team library",
"maintained"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
"family": "paraphrase_trigger",
"score": 0.868,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0,
"lexical_support": 0.267,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this workflow"
],
"eval_optimize": [
"validate",
"route boundary"
],
"reuse_package": [
"package",
"distribution",
"shared capability"
],
"team_operationalize": [
"shared",
"distribution"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
"family": "long_context_trigger",
"score": 0.888,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0.06,
"lexical_support": 0.333,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill",
"skill package"
],
"eval_optimize": [
"validation"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reuse",
"package",
"team reuse"
],
"team_operationalize": [
"team",
"team reuse"
],
"transform_workflow": [
"rough notes",
"transcript",
"onboarding flow"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Give me ideas for improving our process.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"process"
]
},
"negative": {
"brainstorm_only": [
"ideas"
]
}
}
}
},
{
"prompt": "Review this note and tell me what it means in plain English.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"tell me what it means",
"plain english"
]
}
}
}
},
{
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.2,
"negative_penalty": 0.66,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm",
"improve packaging"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"translate_only"
],
"exclusive_negative_concepts": [
"translate_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.273,
"negative_penalty": 0.35,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"translate_only": [
"translate",
"into french"
]
}
}
}
},
{
"prompt": "Summarize this workflow and list the main points only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"summarize_only": [
"summarize"
]
}
}
}
},
{
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.182,
"negative_penalty": 0.68,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
]
},
"negative": {
"explain_only": [
"explain it",
"in simpler language"
],
"no_build_directive": [
"do not convert it into a reusable capability",
"keep it as documentation only"
]
}
}
}
},
{
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
"family": "long_context_summary_only",
"score": 0.257,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.353,
"negative_penalty": 0.3,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"process"
]
},
"negative": {
"summarize_only": [
"summary",
"recap"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"explain_only": [
"explain it"
],
"no_build_directive": [
"no packaging needed"
]
}
}
}
},
{
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.42,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"checklist"
]
},
"negative": {
"no_build_directive": [
"stop short of making a full skill"
]
}
}
}
},
{
"prompt": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document"
],
"no_build_directive": [
"not as an agent skill"
]
}
}
}
},
{
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"iterate_existing_skill": [
"this skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.462,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"reusable"
]
},
"negative": {
"document_only": [
"document",
"reusable-looking document"
],
"no_build_directive": [
"without turning them into an agent skill"
]
}
}
}
},
{
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"family": "long_context_near_neighbor",
"score": 0.328,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.18,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document",
"template only"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.278,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"for later",
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
}
]
}
},
"winner_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 5,
"passed": 5,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
}
},
"family_stats": {
"complex_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset",
"score": 0.468,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0.06,
"lexical_support": 0.316,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable"
],
"team_operationalize": [
"internal"
],
"transform_workflow": [
"process",
"internal process note"
]
},
"negative": {}
}
}
},
{
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill",
"score": 0.729,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.691,
"support_score": 0,
"lexical_support": 0.412,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"near-neighbor",
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"package",
"distribution"
],
"team_operationalize": [
"distribution"
],
"transform_workflow": [
"existing skill draft"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill",
"score": 0.563,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.429,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"validation"
],
"reuse_package": [
"reusable",
"library entry"
],
"transform_workflow": [
"process"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
"family": "paraphrase_trigger",
"score": 0.858,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0.06,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"packaging checks"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"quarterly release routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
"family": "long_context_trigger",
"score": 0.744,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.709,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"turn this playbook into an agent capability package"
],
"reuse_package": [
"package",
"team library"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"prompt history",
"transcript",
"release notes"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.308,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"process"
]
},
"negative": {
"no_build_directive": [
"not a reusable skill"
],
"summarize_only": [
"summary",
"only want a summary"
]
}
}
}
},
{
"prompt": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"package"
],
"transform_workflow": [
"checklist"
]
},
"negative": {
"explain_only": [
"explain this",
"in simpler language"
],
"no_build_directive": [
"do not package it"
]
}
}
}
},
{
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.222,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document"
],
"no_build_directive": [
"not for agent execution"
]
}
}
}
},
{
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"translate_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"no_build_directive": [
"no skill package",
"keep it as documentation only"
],
"translate_only": [
"translate",
"into russian"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"semantic_coverage": 0.309,
"support_score": 0,
"lexical_support": 0.308,
"negative_penalty": 0.46,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
]
},
"negative": {
"brainstorm_only": [
"shape an idea"
],
"future_outline": [
"before we decide whether to build"
]
}
}
}
},
{
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill",
"outline for a possible future skill"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.538,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document",
"document package",
"wiki"
],
"no_build_directive": [
"not an agent skill"
]
}
}
}
},
{
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0.6,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"workflow",
"checklist"
]
},
"negative": {
"document_only": [
"template only"
],
"no_build_directive": [
"no skill package",
"discussion only"
]
}
}
}
}
]
}
},
"current_dev_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 8,
"passed": 8,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 7,
"passed": 7,
"pass_rate": 1.0
}
},
"family_stats": {
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"complex_multi_asset": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_not_package": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"partial_scaffold_not_full_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill",
"score": 0.415,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.5,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"checklist",
"operations checklist"
]
},
"negative": {}
}
}
},
{
"prompt": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill",
"score": 0.388,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.333,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"trigger evals"
],
"iterate_existing_skill": [
"this skill",
"sharing it with the team",
"before sharing it with the team"
],
"team_operationalize": [
"team",
"sharing it with the team"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset",
"score": 0.843,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package",
"turn all of that into one reusable skill package"
],
"eval_optimize": [
"evals",
"packaging checks"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"runbook",
"process",
"prompt history"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset",
"score": 0.418,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.357,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"production-ready"
],
"transform_workflow": [
"workflow",
"workflow fragments"
]
},
"negative": {}
}
}
},
{
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill",
"score": 0.557,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"distribution"
],
"team_operationalize": [
"team",
"distribution"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
"family": "paraphrase_trigger",
"score": 0.471,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0,
"lexical_support": 0.538,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"reuse_package": [
"team library",
"maintained"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
"family": "paraphrase_trigger",
"score": 0.868,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0,
"lexical_support": 0.267,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this workflow"
],
"eval_optimize": [
"validate",
"route boundary"
],
"reuse_package": [
"package",
"distribution",
"shared capability"
],
"team_operationalize": [
"shared",
"distribution"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
"family": "long_context_trigger",
"score": 0.888,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.855,
"support_score": 0.06,
"lexical_support": 0.333,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill",
"skill package"
],
"eval_optimize": [
"validation"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reuse",
"package",
"team reuse"
],
"team_operationalize": [
"team",
"team reuse"
],
"transform_workflow": [
"rough notes",
"transcript",
"onboarding flow"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Give me ideas for improving our process.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"process"
]
},
"negative": {
"brainstorm_only": [
"ideas"
]
}
}
}
},
{
"prompt": "Review this note and tell me what it means in plain English.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"tell me what it means",
"plain english"
]
}
}
}
},
{
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.2,
"negative_penalty": 0.66,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm",
"improve packaging"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"translate_only"
],
"exclusive_negative_concepts": [
"translate_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.273,
"negative_penalty": 0.35,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"translate_only": [
"translate",
"into french"
]
}
}
}
},
{
"prompt": "Summarize this workflow and list the main points only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"summarize_only": [
"summarize"
]
}
}
}
},
{
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.182,
"negative_penalty": 0.68,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
]
},
"negative": {
"explain_only": [
"explain it",
"in simpler language"
],
"no_build_directive": [
"do not convert it into a reusable capability",
"keep it as documentation only"
]
}
}
}
},
{
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
"family": "long_context_summary_only",
"score": 0.257,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"summarize_only"
],
"exclusive_negative_concepts": [
"summarize_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.353,
"negative_penalty": 0.3,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"process"
]
},
"negative": {
"summarize_only": [
"summary",
"recap"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"explain_only": [
"explain it"
],
"no_build_directive": [
"no packaging needed"
]
}
}
}
},
{
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.42,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"checklist"
]
},
"negative": {
"no_build_directive": [
"stop short of making a full skill"
]
}
}
}
},
{
"prompt": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document"
],
"no_build_directive": [
"not as an agent skill"
]
}
}
}
},
{
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"iterate_existing_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.327,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"iterate_existing_skill": [
"this skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.462,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"reusable"
]
},
"negative": {
"document_only": [
"document",
"reusable-looking document"
],
"no_build_directive": [
"without turning them into an agent skill"
]
}
}
}
},
{
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
"family": "long_context_near_neighbor",
"score": 0.328,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.18,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document",
"template only"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.278,
"negative_penalty": 0.64,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
]
},
"negative": {
"future_outline": [
"for later",
"future-ready outline"
],
"no_build_directive": [
"do not build"
]
}
}
}
}
]
}
},
"current_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 5,
"passed": 5,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 4,
"passed": 4,
"pass_rate": 1.0
}
},
"family_stats": {
"complex_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"iterate_existing_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"workflow_to_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"paraphrase_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_trigger": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"translate_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"future_outline_vs_build": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"document_export_vs_agent_skill": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"long_context_near_neighbor": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset",
"score": 0.468,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.4,
"support_score": 0.06,
"lexical_support": 0.316,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable"
],
"team_operationalize": [
"internal"
],
"transform_workflow": [
"process",
"internal process note"
]
},
"negative": {}
}
}
},
{
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill",
"score": 0.729,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"iterate_existing_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.691,
"support_score": 0,
"lexical_support": 0.412,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"evals",
"near-neighbor",
"trigger boundary"
],
"iterate_existing_skill": [
"existing skill"
],
"reuse_package": [
"package",
"distribution"
],
"team_operationalize": [
"distribution"
],
"transform_workflow": [
"existing skill draft"
]
},
"negative": {}
}
}
},
{
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill",
"score": 0.563,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.509,
"support_score": 0,
"lexical_support": 0.429,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"eval_optimize": [
"validation"
],
"reuse_package": [
"reusable",
"library entry"
],
"transform_workflow": [
"process"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
"family": "paraphrase_trigger",
"score": 0.858,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0.06,
"lexical_support": 0.375,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"packaging checks"
],
"multi_asset": [
"references",
"scripts"
],
"reuse_package": [
"reusable",
"package",
"packaging"
],
"transform_workflow": [
"quarterly release routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
"family": "long_context_trigger",
"score": 0.744,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"team_operationalize",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.709,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"turn this playbook into an agent capability package"
],
"reuse_package": [
"package",
"team library"
],
"team_operationalize": [
"team"
],
"transform_workflow": [
"prompt history",
"transcript",
"release notes"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.308,
"negative_penalty": 0.72,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable"
],
"transform_workflow": [
"process"
]
},
"negative": {
"no_build_directive": [
"not a reusable skill"
],
"summarize_only": [
"summary",
"only want a summary"
]
}
}
}
},
{
"prompt": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"explain_only",
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.68,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"package"
],
"transform_workflow": [
"checklist"
]
},
"negative": {
"explain_only": [
"explain this",
"in simpler language"
],
"no_build_directive": [
"do not package it"
]
}
}
}
},
{
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
"family": "long_context_document_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.222,
"negative_penalty": 0.6,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"document",
"wiki",
"reference document"
],
"no_build_directive": [
"not for agent execution"
]
}
}
}
},
{
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
"family": "translate_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"no_build_directive",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"translate_only"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.77,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"no_build_directive": [
"no skill package",
"keep it as documentation only"
],
"translate_only": [
"translate",
"into russian"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline"
],
"semantic_coverage": 0.309,
"support_score": 0,
"lexical_support": 0.308,
"negative_penalty": 0.46,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"build_skill": [
"build a skill"
]
},
"negative": {
"brainstorm_only": [
"shape an idea"
],
"future_outline": [
"before we decide whether to build"
]
}
}
}
},
{
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.357,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill",
"outline for a possible future skill"
],
"no_build_directive": [
"do not build"
]
}
}
}
},
{
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.491,
"support_score": 0,
"lexical_support": 0.538,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"build_skill": [
"agent skill"
],
"reuse_package": [
"package"
]
},
"negative": {
"document_only": [
"document",
"document package",
"wiki"
],
"no_build_directive": [
"not an agent skill"
]
}
}
}
},
{
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
"family": "long_context_near_neighbor",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.673,
"support_score": 0,
"lexical_support": 0.375,
"negative_penalty": 0.6,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"workflow",
"checklist"
]
},
"negative": {
"document_only": [
"template only"
],
"no_build_directive": [
"no skill package",
"discussion only"
]
}
}
}
}
]
}
},
"winner_blind_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"score": 0.392,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.214,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"score": 0.407,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"score": 0.024,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.273,
"negative_penalty": 0.18,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
}
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.3,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.0,
"support_score": 0.06,
"lexical_support": 0.308,
"negative_penalty": 0.22,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
}
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"score": 0.026,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.273,
"negative_penalty": 0.22,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
}
}
]
}
},
"current_blind_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"score": 0.392,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.214,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"score": 0.407,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"score": 0.024,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.182,
"support_score": 0.06,
"lexical_support": 0.273,
"negative_penalty": 0.18,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
}
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.3,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.0,
"support_score": 0.06,
"lexical_support": 0.308,
"negative_penalty": 0.22,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
}
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"score": 0.026,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.364,
"support_score": 0,
"lexical_support": 0.273,
"negative_penalty": 0.22,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
}
}
]
}
},
"baseline_blind_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"score": 0.393,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.071,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"score": 0.393,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.067,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"score": 0.016,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only"
],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.189,
"support_score": 0.06,
"lexical_support": 0.091,
"negative_penalty": 0.18,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
}
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.0,
"support_score": 0.06,
"lexical_support": 0.0,
"negative_penalty": 0.22,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
}
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"score": 0.024,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline"
],
"exclusive_negative_concepts": [
"future_outline"
],
"semantic_coverage": 0.377,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.22,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
}
}
]
}
},
"winner_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 0,
"margin": 3,
"confidence": 0.74,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow"
],
"strong_positive_concepts": [
"transform_workflow"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [
"document_only"
],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 2,
"negative_vote": 1,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"prompts"
],
"artifact_hits": [
"prompts"
],
"capability_overlap": 2,
"exclusive_negative_concepts": [
"explain_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain",
"plain english"
],
"positive_vote": 2,
"negative_vote": 3,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 2,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 2,
"margin": 1,
"confidence": 0.66,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
},
"boundary_case": true
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"current_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 0,
"margin": 3,
"confidence": 0.74,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow"
],
"strong_positive_concepts": [
"transform_workflow"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [
"document_only"
],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 2,
"negative_vote": 1,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"prompts"
],
"artifact_hits": [
"prompts"
],
"capability_overlap": 2,
"exclusive_negative_concepts": [
"explain_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain",
"plain english"
],
"positive_vote": 2,
"negative_vote": 3,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 2,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 2,
"margin": 1,
"confidence": 0.66,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
},
"boundary_case": true
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"baseline_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_multi_asset": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_document_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_future_outline": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
"family": "blind_operationalize",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 0,
"margin": 3,
"confidence": 0.74,
"concept_evidence": {
"positive": {
"reuse_package": [
"maintained",
"operationalize"
],
"transform_workflow": [
"support escalation routine"
]
},
"negative": {}
}
}
},
{
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
"family": "blind_multi_asset",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"reuse_package": [
"reusable",
"package"
],
"transform_workflow": [
"sop",
"transcript"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
"family": "blind_document_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow"
],
"strong_positive_concepts": [
"transform_workflow"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [
"document_only"
],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 2,
"negative_vote": 1,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"transform_workflow": [
"workflow"
]
},
"negative": {
"document_only": [
"wiki"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Explain the differences between prompts and skills in plain English.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"prompts"
],
"artifact_hits": [
"prompts"
],
"capability_overlap": 2,
"exclusive_negative_concepts": [
"explain_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain",
"plain english"
],
"positive_vote": 2,
"negative_vote": 3,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"explain_only": [
"plain english"
]
}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
"family": "blind_future_outline",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [
"notes"
],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 2,
"margin": -1,
"confidence": 0.58,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
]
},
"negative": {
"future_outline": [
"future skill"
]
}
}
},
"boundary_case": true
},
{
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"strong_positive_concepts": [
"transform_workflow",
"reuse_package"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"future_outline"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 3,
"negative_vote": 2,
"margin": 1,
"confidence": 0.66,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
],
"transform_workflow": [
"process"
]
},
"negative": {
"future_outline": [
"for later"
]
}
}
},
"boundary_case": true
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"winner_adversarial_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_noisy_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_multi_asset_boundary": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_document_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_summary_translation_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_outline_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_brainstorm_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
"family": "adversarial_noisy_operationalize",
"score": 0.834,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.263,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"hardening"
],
"reuse_package": [
"package",
"maintained",
"operationalize"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
"family": "adversarial_multi_asset_boundary",
"score": 0.842,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.368,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this routine"
],
"eval_optimize": [
"stress test",
"route boundary"
],
"reuse_package": [
"reusable"
],
"transform_workflow": [
"prompt history"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
"family": "adversarial_document_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.429,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"document_only": [
"wiki"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
"family": "adversarial_summary_translation_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive",
"summarize_only",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only",
"translate_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 1.25,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"sop"
]
},
"negative": {
"document_only": [
"knowledge base"
],
"no_build_directive": [
"keep it as documentation only"
],
"summarize_only": [
"summarize"
],
"translate_only": [
"translate",
"into japanese"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
"family": "adversarial_future_outline_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill"
],
"no_build_directive": [
"stop short of making a full skill",
"plain note",
"keep it as a plain note"
]
}
}
}
},
{
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
"family": "adversarial_brainstorm_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.182,
"negative_penalty": 0.88,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
],
"future_outline": [
"for later"
],
"no_build_directive": [
"discussion only"
]
}
}
}
}
]
}
},
"current_adversarial_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_noisy_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_multi_asset_boundary": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_document_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_summary_translation_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_outline_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_brainstorm_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
"family": "adversarial_noisy_operationalize",
"score": 0.834,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.263,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"hardening"
],
"reuse_package": [
"package",
"maintained",
"operationalize"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
"family": "adversarial_multi_asset_boundary",
"score": 0.842,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.818,
"support_score": 0,
"lexical_support": 0.368,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this routine"
],
"eval_optimize": [
"stress test",
"route boundary"
],
"reuse_package": [
"reusable"
],
"transform_workflow": [
"prompt history"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
"family": "adversarial_document_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.364,
"support_score": 0.06,
"lexical_support": 0.429,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"document_only": [
"wiki"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
"family": "adversarial_summary_translation_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive",
"summarize_only",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only",
"translate_only"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.2,
"negative_penalty": 1.25,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"sop"
]
},
"negative": {
"document_only": [
"knowledge base"
],
"no_build_directive": [
"keep it as documentation only"
],
"summarize_only": [
"summarize"
],
"translate_only": [
"translate",
"into japanese"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
"family": "adversarial_future_outline_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.25,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill"
],
"no_build_directive": [
"stop short of making a full skill",
"plain note",
"keep it as a plain note"
]
}
}
}
},
{
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
"family": "adversarial_brainstorm_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill",
"team_operationalize"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.182,
"support_score": 0,
"lexical_support": 0.182,
"negative_penalty": 0.88,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
],
"future_outline": [
"for later"
],
"no_build_directive": [
"discussion only"
]
}
}
}
}
]
}
},
"baseline_adversarial_holdout_report": {
"threshold": 0.33,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_noisy_operationalize": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_multi_asset_boundary": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_document_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_summary_translation_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_outline_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_brainstorm_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
"family": "adversarial_noisy_operationalize",
"score": 0.845,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0,
"lexical_support": 0.053,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"skill package"
],
"eval_optimize": [
"hardening"
],
"reuse_package": [
"package",
"maintained",
"operationalize"
],
"transform_workflow": [
"workflow"
]
},
"negative": {}
}
}
},
{
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
"family": "adversarial_multi_asset_boundary",
"score": 0.845,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"build_skill",
"eval_optimize",
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.849,
"support_score": 0,
"lexical_support": 0.053,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"build_skill": [
"codify this routine"
],
"eval_optimize": [
"stress test",
"route boundary"
],
"reuse_package": [
"reusable"
],
"transform_workflow": [
"prompt history"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
"family": "adversarial_document_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package",
"transform_workflow"
],
"extra_positive_concepts": [
"multi_asset"
],
"matched_negative_concepts": [
"document_only",
"no_build_directive"
],
"exclusive_negative_concepts": [
"no_build_directive"
],
"semantic_coverage": 0.377,
"support_score": 0.06,
"lexical_support": 0.143,
"negative_penalty": 0.6,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"multi_asset": [
"files"
],
"reuse_package": [
"package"
],
"transform_workflow": [
"runbook"
]
},
"negative": {
"document_only": [
"wiki"
],
"no_build_directive": [
"do not generate any skill files"
]
}
}
}
},
{
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
"family": "adversarial_summary_translation_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"transform_workflow"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"document_only",
"no_build_directive",
"summarize_only",
"translate_only"
],
"exclusive_negative_concepts": [
"no_build_directive",
"summarize_only",
"translate_only"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.067,
"negative_penalty": 1.25,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"transform_workflow": [
"sop"
]
},
"negative": {
"document_only": [
"knowledge base"
],
"no_build_directive": [
"keep it as documentation only"
],
"summarize_only": [
"summarize"
],
"translate_only": [
"translate",
"into japanese"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
"family": "adversarial_future_outline_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.05,
"negative_penalty": 0.64,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"future_outline": [
"future skill",
"possible future skill"
],
"no_build_directive": [
"stop short of making a full skill",
"plain note",
"keep it as a plain note"
]
}
}
}
},
{
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
"family": "adversarial_brainstorm_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"build_skill",
"transform_workflow",
"reuse_package",
"eval_optimize",
"iterate_existing_skill"
],
"matched_desired_concepts": [
"reuse_package"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"exclusive_negative_concepts": [
"brainstorm_only",
"future_outline",
"no_build_directive"
],
"semantic_coverage": 0.189,
"support_score": 0,
"lexical_support": 0.091,
"negative_penalty": 0.88,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reuse_package": [
"packaging"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
],
"future_outline": [
"for later"
],
"no_build_directive": [
"discussion only"
]
}
}
}
}
]
}
},
"candidates": [
{
"id": "current",
"label": "Current",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
"strategy": "current",
"estimated_tokens": 65,
"dev": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
0,
-1.0,
-1.0,
-1.0,
-1.0,
65
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "minimal",
"label": "Minimal",
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"strategy": "minimal_template",
"estimated_tokens": 41,
"dev": {
"false_positives": 2,
"false_negatives": 1,
"precision": 0.778,
"recall": 0.875,
"near_neighbor_pass_rate": 0.857,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
2,
1,
-0.857,
-0.857,
-0.778,
-0.875,
41
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "balanced",
"label": "Balanced",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"strategy": "balanced_template",
"estimated_tokens": 54,
"dev": {
"false_positives": 2,
"false_negatives": 1,
"precision": 0.778,
"recall": 0.875,
"near_neighbor_pass_rate": 0.857,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
2,
1,
-0.857,
-0.857,
-0.778,
-0.875,
54
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "artifact_aware",
"label": "Artifact Aware",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
"strategy": "artifact_template",
"estimated_tokens": 72,
"dev": {
"false_positives": 2,
"false_negatives": 1,
"precision": 0.778,
"recall": 0.875,
"near_neighbor_pass_rate": 0.857,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
2,
1,
-0.857,
-0.857,
-0.778,
-0.875,
72
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "boundary",
"label": "Boundary",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"strategy": "boundary_template",
"estimated_tokens": 78,
"dev": {
"false_positives": 2,
"false_negatives": 1,
"precision": 0.778,
"recall": 0.875,
"near_neighbor_pass_rate": 0.857,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
2,
1,
-0.857,
-0.857,
-0.778,
-0.875,
78
],
"holdout": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "guardrail",
"label": "Guardrail",
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
"strategy": "guardrail_template",
"estimated_tokens": 50,
"dev": {
"false_positives": 5,
"false_negatives": 1,
"precision": 0.583,
"recall": 0.875,
"near_neighbor_pass_rate": 0.429,
"should_not_trigger_pass_rate": 0.857
},
"selection_key": [
5,
1,
-0.429,
-0.857,
-0.583,
-0.875,
50
],
"holdout": {
"false_positives": 2,
"false_negatives": 0,
"precision": 0.714,
"recall": 1.0,
"near_neighbor_pass_rate": 0.5,
"should_not_trigger_pass_rate": 1.0
}
}
],
"selection_logic": {
"priority": [
"fewest false positives",
"fewest false negatives",
"highest near-neighbor pass rate",
"highest negative pass rate",
"highest precision",
"highest recall",
"shortest description"
]
},
"comparison": {
"winner_vs_current_dev": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_dev": {
"baseline_false_positives": 1,
"baseline_false_negatives": 1,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": -1,
"false_negative_delta": -1,
"baseline_precision": 0.875,
"improved_precision": 1.0,
"baseline_recall": 0.875,
"improved_recall": 1.0
},
"winner_vs_baseline_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_judge_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_judge_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_adversarial_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_adversarial_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
}
},
"acceptance_gates": {
"selection_basis": "dev only",
"holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.672,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
"max_non_trigger_score": 0.0,
"score_gap": 0.468,
"threshold_margin": 0.138,
"positive_threshold_buffer": 0.138,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"current_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.672,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.468,
"max_non_trigger_score": 0.0,
"score_gap": 0.468,
"threshold_margin": 0.138,
"positive_threshold_buffer": 0.138,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"baseline_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.648,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.416,
"max_non_trigger_score": 0.0,
"score_gap": 0.416,
"threshold_margin": 0.086,
"positive_threshold_buffer": 0.086,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"winner_family_health": {
"family_count": 12,
"clean_family_count": 12,
"failing_families": [],
"weakest_family": {
"family": "workflow_to_skill",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 12,
"clean_family_count": 12,
"failing_families": [],
"weakest_family": {
"family": "workflow_to_skill",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 12,
"clean_family_count": 12,
"failing_families": [],
"weakest_family": {
"family": "workflow_to_skill",
"pass_rate": 1.0,
"errors": 0
}
}
},
"blind_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.399,
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.026,
"score_gap": 0.366,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.304,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
},
"current_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.399,
"mean_non_trigger_score": 0.013,
"mean_near_neighbor_score": 0.013,
"min_positive_score": 0.392,
"max_non_trigger_score": 0.026,
"score_gap": 0.366,
"threshold_margin": 0.062,
"positive_threshold_buffer": 0.062,
"negative_threshold_buffer": 0.304,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
},
"baseline_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.393,
"mean_non_trigger_score": 0.01,
"mean_near_neighbor_score": 0.012,
"min_positive_score": 0.393,
"max_non_trigger_score": 0.024,
"score_gap": 0.369,
"threshold_margin": 0.063,
"positive_threshold_buffer": 0.063,
"negative_threshold_buffer": 0.306,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "watch"
},
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
}
},
"judge_blind_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.66,
"rubric_version": "blind-v1"
}
},
"winner_calibration": null,
"current_calibration": null,
"baseline_calibration": null,
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_operationalize",
"pass_rate": 1.0,
"errors": 0
}
}
},
"adversarial_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.838,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.834,
"max_non_trigger_score": 0.0,
"score_gap": 0.834,
"threshold_margin": 0.33,
"positive_threshold_buffer": 0.504,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"current_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.838,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.834,
"max_non_trigger_score": 0.0,
"score_gap": 0.834,
"threshold_margin": 0.33,
"positive_threshold_buffer": 0.504,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"baseline_calibration": {
"threshold": 0.33,
"mean_positive_score": 0.845,
"mean_non_trigger_score": 0.0,
"mean_near_neighbor_score": 0.0,
"min_positive_score": 0.845,
"max_non_trigger_score": 0.0,
"score_gap": 0.845,
"threshold_margin": 0.33,
"positive_threshold_buffer": 0.515,
"negative_threshold_buffer": 0.33,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_summary_translation_collision",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_summary_translation_collision",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_summary_translation_collision",
"pass_rate": 1.0,
"errors": 0
}
}
}
},
"summary": {
"winner_label": "Current",
"winner_tokens": 65,
"current_tokens": 65,
"winner_dev_total_errors": 0,
"current_dev_total_errors": 0,
"winner_holdout_total_errors": 0,
"current_holdout_total_errors": 0,
"winner_blind_holdout_total_errors": 0,
"current_blind_holdout_total_errors": 0,
"winner_judge_blind_holdout_total_errors": 0,
"current_judge_blind_holdout_total_errors": 0,
"winner_adversarial_holdout_total_errors": 0,
"current_adversarial_holdout_total_errors": 0,
"winner_judge_blind_agreement_rate": 1.0,
"winner_adversarial_risk_band": "healthy",
"winner_adversarial_score_gap": 0.834,
"candidate_count": 6,
"baseline_tokens": 8,
"baseline_dev_total_errors": 2,
"baseline_holdout_total_errors": 0,
"baseline_blind_holdout_total_errors": 0,
"baseline_judge_blind_holdout_total_errors": 0,
"baseline_adversarial_holdout_total_errors": 0
}
}
Reference in New Issue View Git Blame Copy Permalink