10521 lines
323 KiB
JSON
10521 lines
323 KiB
JSON
{
|
|
"current_description": "Create/improve/evaluate agent skills from workflows, prompts, SOPs, scripts. Use for migration/release/package, routing, evals/tests, install checks, 优化已有 skill, 补 trigger 评测. Exclude no-skill summary/translation/docs.",
|
|
"current_candidate": {
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Create/improve/evaluate agent skills from workflows, prompts, SOPs, scripts. Use for migration/release/package, routing, evals/tests, install checks, 优化已有 skill, 补 trigger 评测. Exclude no-skill summary/translation/docs.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 54,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
54
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
"baseline": {
|
|
"description": "Create and improve agent skills.",
|
|
"estimated_tokens": 8,
|
|
"dev": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.875,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 8,
|
|
"passed": 7,
|
|
"pass_rate": 0.875
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 7,
|
|
"passed": 7,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 7,
|
|
"passed": 7,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"workflow_to_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"iterate_existing_skill": {
|
|
"total": 2,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.5
|
|
},
|
|
"complex_multi_asset": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"paraphrase_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"translate_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_not_package": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"partial_scaffold_not_full_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"document_export_vs_agent_skill": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"future_outline_vs_build": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_near_neighbor": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "iterate_existing_skill",
|
|
"prompt": "Add trigger evals to this skill before sharing it with the team.",
|
|
"score": 0.328,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Convert this operations checklist into a reusable skill.",
|
|
"family": "workflow_to_skill",
|
|
"score": 0.387,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"checklist",
|
|
"operations checklist"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Add trigger evals to this skill before sharing it with the team.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.328,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.302,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"evals",
|
|
"trigger evals"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"this skill",
|
|
"sharing it with the team",
|
|
"before sharing it with the team"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"sharing it with the team"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.844,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.849,
|
|
"support_score": 0,
|
|
"lexical_support": 0.042,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package",
|
|
"turn all of that into one reusable skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"evals",
|
|
"packaging checks"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package",
|
|
"packaging",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook",
|
|
"process",
|
|
"prompt history"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.408,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0.06,
|
|
"lexical_support": 0.067,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"production-ready"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow",
|
|
"workflow fragments",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.527,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.491,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"trigger boundary"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"existing skill"
|
|
],
|
|
"reuse_package": [
|
|
"distribution"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"distribution"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.403,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"team library",
|
|
"maintained"
|
|
],
|
|
"team_operationalize": [
|
|
"team"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.856,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.849,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.067,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"codify this workflow"
|
|
],
|
|
"eval_optimize": [
|
|
"validate",
|
|
"route boundary"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"distribution",
|
|
"shared capability"
|
|
],
|
|
"team_operationalize": [
|
|
"shared",
|
|
"distribution"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
|
|
"family": "long_context_trigger",
|
|
"score": 0.869,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"multi_asset",
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.849,
|
|
"support_score": 0.1,
|
|
"lexical_support": 0.038,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill",
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"validation"
|
|
],
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reuse",
|
|
"package",
|
|
"team reuse"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"team reuse"
|
|
],
|
|
"transform_workflow": [
|
|
"rough notes",
|
|
"transcript",
|
|
"onboarding flow",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Give me ideas for improving our process.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"ideas"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Review this note and tell me what it means in plain English.",
|
|
"family": "explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"tell me what it means",
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0.06,
|
|
"lexical_support": 0.067,
|
|
"negative_penalty": 0.66,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"reuse_package": [
|
|
"packaging"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm",
|
|
"improve packaging"
|
|
],
|
|
"no_build_directive": [
|
|
"do not generate any skill files"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
|
|
"family": "translate_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0.35,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"translate_only": [
|
|
"translate",
|
|
"into french"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Summarize this workflow and list the main points only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0,
|
|
"lexical_support": 0.111,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"summarize_only": [
|
|
"summarize"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
|
|
"family": "long_context_document_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0,
|
|
"lexical_support": 0.043,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain it",
|
|
"in simpler language"
|
|
],
|
|
"no_build_directive": [
|
|
"do not convert it into a reusable capability",
|
|
"keep it as documentation only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
|
|
"family": "long_context_summary_only",
|
|
"score": 0.257,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.698,
|
|
"support_score": 0,
|
|
"lexical_support": 0.056,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"summarize_only": [
|
|
"summary",
|
|
"recap"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Review this process note and explain it, no packaging needed.",
|
|
"family": "explain_not_package",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain it"
|
|
],
|
|
"no_build_directive": [
|
|
"no packaging needed"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
|
|
"family": "partial_scaffold_not_full_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0.42,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"stop short of making a full skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Package this explanation as a document, not as an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.509,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document"
|
|
],
|
|
"no_build_directive": [
|
|
"not as an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"iterate_existing_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.34,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"iterate_existing_skill": [
|
|
"this skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future-ready outline"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.509,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"reusable-looking document"
|
|
],
|
|
"no_build_directive": [
|
|
"without turning them into an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
|
|
"family": "long_context_near_neighbor",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.509,
|
|
"support_score": 0,
|
|
"lexical_support": 0.048,
|
|
"negative_penalty": 0.82,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"wiki",
|
|
"reference document",
|
|
"template only"
|
|
],
|
|
"future_outline": [
|
|
"not now",
|
|
"later but not now"
|
|
],
|
|
"no_build_directive": [
|
|
"not now",
|
|
"later but not now"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.509,
|
|
"support_score": 0,
|
|
"lexical_support": 0.053,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later",
|
|
"future-ready outline"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"holdout": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 5,
|
|
"passed": 5,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 4,
|
|
"passed": 4,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 4,
|
|
"passed": 4,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"complex_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"iterate_existing_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"workflow_to_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"paraphrase_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"translate_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"future_outline_vs_build": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"document_export_vs_agent_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_near_neighbor": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.416,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"multi_asset",
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0.1,
|
|
"lexical_support": 0.053,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"team_operationalize": [
|
|
"internal"
|
|
],
|
|
"transform_workflow": [
|
|
"process",
|
|
"internal process note",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.699,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.679,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.056,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"evals",
|
|
"near-neighbor",
|
|
"trigger boundary"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"existing skill"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"distribution"
|
|
],
|
|
"team_operationalize": [
|
|
"distribution"
|
|
],
|
|
"transform_workflow": [
|
|
"existing skill draft"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
|
|
"family": "workflow_to_skill",
|
|
"score": 0.841,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.849,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill library entry"
|
|
],
|
|
"eval_optimize": [
|
|
"validation"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"library entry"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.861,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.849,
|
|
"support_score": 0.06,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"packaging checks"
|
|
],
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package",
|
|
"packaging",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"quarterly release routine",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
|
|
"family": "long_context_trigger",
|
|
"score": 0.719,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.698,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.08,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"turn this playbook into an agent capability package",
|
|
"agent capability"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"team library",
|
|
"release"
|
|
],
|
|
"team_operationalize": [
|
|
"team"
|
|
],
|
|
"transform_workflow": [
|
|
"prompt history",
|
|
"playbook",
|
|
"transcript",
|
|
"release notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.72,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"not a reusable skill"
|
|
],
|
|
"summarize_only": [
|
|
"summary",
|
|
"only want a summary"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain this release checklist in simpler language; do not package it.",
|
|
"family": "explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"package",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain this",
|
|
"in simpler language"
|
|
],
|
|
"no_build_directive": [
|
|
"do not package it"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
|
|
"family": "long_context_document_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.111,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"wiki",
|
|
"reference document"
|
|
],
|
|
"no_build_directive": [
|
|
"not for agent execution"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
|
|
"family": "translate_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive",
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.698,
|
|
"support_score": 0,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 0.77,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"no skill package",
|
|
"keep it as documentation only"
|
|
],
|
|
"translate_only": [
|
|
"translate",
|
|
"into russian"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Help me shape an idea before we decide whether to build a skill.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.321,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.46,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"shape an idea"
|
|
],
|
|
"future_outline": [
|
|
"before we decide whether to build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill",
|
|
"possible future skill",
|
|
"outline for a possible future skill"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.509,
|
|
"support_score": 0,
|
|
"lexical_support": 0.077,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"document package",
|
|
"wiki"
|
|
],
|
|
"no_build_directive": [
|
|
"not an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
|
|
"family": "long_context_near_neighbor",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.698,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow",
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"template only"
|
|
],
|
|
"no_build_directive": [
|
|
"no skill package",
|
|
"discussion only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"winner": {
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Create/improve/evaluate agent skills from workflows, prompts, SOPs, scripts. Use for migration/release/package, routing, evals/tests, install checks, 优化已有 skill, 补 trigger 评测. Exclude no-skill summary/translation/docs.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 54,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
54
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
"winner_dev_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 8,
|
|
"passed": 8,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 7,
|
|
"passed": 7,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 7,
|
|
"passed": 7,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"workflow_to_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"iterate_existing_skill": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"complex_multi_asset": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"paraphrase_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"translate_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_not_package": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"partial_scaffold_not_full_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"document_export_vs_agent_skill": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"future_outline_vs_build": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_near_neighbor": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Convert this operations checklist into a reusable skill.",
|
|
"family": "workflow_to_skill",
|
|
"score": 0.379,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"checklist",
|
|
"operations checklist"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Add trigger evals to this skill before sharing it with the team.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.333,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.286,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"evals",
|
|
"trigger evals"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"this skill",
|
|
"sharing it with the team",
|
|
"before sharing it with the team"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"sharing it with the team"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.816,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.208,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package",
|
|
"turn all of that into one reusable skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"evals",
|
|
"packaging checks"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package",
|
|
"packaging",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook",
|
|
"process",
|
|
"prompt history"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.454,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.411,
|
|
"support_score": 0,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"production-ready"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow",
|
|
"workflow fragments",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.514,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.464,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.214,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"trigger boundary"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"existing skill"
|
|
],
|
|
"reuse_package": [
|
|
"distribution"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"distribution"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.39,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"team library",
|
|
"maintained"
|
|
],
|
|
"team_operationalize": [
|
|
"team"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.815,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.067,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"codify this workflow"
|
|
],
|
|
"eval_optimize": [
|
|
"validate",
|
|
"route boundary"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"distribution",
|
|
"shared capability"
|
|
],
|
|
"team_operationalize": [
|
|
"shared",
|
|
"distribution"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
|
|
"family": "long_context_trigger",
|
|
"score": 0.871,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.857,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.154,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill",
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"validation"
|
|
],
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reuse",
|
|
"package",
|
|
"team reuse"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"team reuse"
|
|
],
|
|
"transform_workflow": [
|
|
"rough notes",
|
|
"transcript",
|
|
"onboarding flow",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Give me ideas for improving our process.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"ideas"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Review this note and tell me what it means in plain English.",
|
|
"family": "explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"tell me what it means",
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.232,
|
|
"support_score": 0,
|
|
"lexical_support": 0.133,
|
|
"negative_penalty": 0.66,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"reuse_package": [
|
|
"packaging"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm",
|
|
"improve packaging"
|
|
],
|
|
"no_build_directive": [
|
|
"do not generate any skill files"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
|
|
"family": "translate_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.35,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"translate_only": [
|
|
"translate",
|
|
"into french"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Summarize this workflow and list the main points only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"summarize_only": [
|
|
"summarize"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
|
|
"family": "long_context_document_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain it",
|
|
"in simpler language"
|
|
],
|
|
"no_build_directive": [
|
|
"do not convert it into a reusable capability",
|
|
"keep it as documentation only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
|
|
"family": "long_context_summary_only",
|
|
"score": 0.236,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"summarize_only": [
|
|
"summary",
|
|
"recap"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Review this process note and explain it, no packaging needed.",
|
|
"family": "explain_not_package",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain it"
|
|
],
|
|
"no_build_directive": [
|
|
"no packaging needed"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
|
|
"family": "partial_scaffold_not_full_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0.42,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"stop short of making a full skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Package this explanation as a document, not as an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.3,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document"
|
|
],
|
|
"no_build_directive": [
|
|
"not as an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"iterate_existing_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.321,
|
|
"support_score": 0,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"iterate_existing_skill": [
|
|
"this skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future-ready outline"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"reusable-looking document"
|
|
],
|
|
"no_build_directive": [
|
|
"without turning them into an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
|
|
"family": "long_context_near_neighbor",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.048,
|
|
"negative_penalty": 0.82,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"wiki",
|
|
"reference document",
|
|
"template only"
|
|
],
|
|
"future_outline": [
|
|
"not now",
|
|
"later but not now"
|
|
],
|
|
"no_build_directive": [
|
|
"not now",
|
|
"later but not now"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.158,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later",
|
|
"future-ready outline"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"winner_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 5,
|
|
"passed": 5,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 4,
|
|
"passed": 4,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 4,
|
|
"passed": 4,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"complex_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"iterate_existing_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"workflow_to_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"paraphrase_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"translate_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"future_outline_vs_build": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"document_export_vs_agent_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_near_neighbor": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.465,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.411,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.211,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"team_operationalize": [
|
|
"internal"
|
|
],
|
|
"transform_workflow": [
|
|
"process",
|
|
"internal process note",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.684,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.643,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.278,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"evals",
|
|
"near-neighbor",
|
|
"trigger boundary"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"existing skill"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"distribution"
|
|
],
|
|
"team_operationalize": [
|
|
"distribution"
|
|
],
|
|
"transform_workflow": [
|
|
"existing skill draft"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
|
|
"family": "workflow_to_skill",
|
|
"score": 0.805,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill library entry"
|
|
],
|
|
"eval_optimize": [
|
|
"validation"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"library entry"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.874,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.857,
|
|
"support_score": 0,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"packaging checks"
|
|
],
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package",
|
|
"packaging",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"quarterly release routine",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
|
|
"family": "long_context_trigger",
|
|
"score": 0.691,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.16,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"turn this playbook into an agent capability package",
|
|
"agent capability"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"team library",
|
|
"release"
|
|
],
|
|
"team_operationalize": [
|
|
"team"
|
|
],
|
|
"transform_workflow": [
|
|
"prompt history",
|
|
"playbook",
|
|
"transcript",
|
|
"release notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.72,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"not a reusable skill"
|
|
],
|
|
"summarize_only": [
|
|
"summary",
|
|
"only want a summary"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain this release checklist in simpler language; do not package it.",
|
|
"family": "explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.182,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"package",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain this",
|
|
"in simpler language"
|
|
],
|
|
"no_build_directive": [
|
|
"do not package it"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
|
|
"family": "long_context_document_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.167,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"wiki",
|
|
"reference document"
|
|
],
|
|
"no_build_directive": [
|
|
"not for agent execution"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
|
|
"family": "translate_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive",
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0,
|
|
"lexical_support": 0.188,
|
|
"negative_penalty": 0.77,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"no skill package",
|
|
"keep it as documentation only"
|
|
],
|
|
"translate_only": [
|
|
"translate",
|
|
"into russian"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Help me shape an idea before we decide whether to build a skill.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.304,
|
|
"support_score": 0,
|
|
"lexical_support": 0.077,
|
|
"negative_penalty": 0.46,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"shape an idea"
|
|
],
|
|
"future_outline": [
|
|
"before we decide whether to build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.214,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill",
|
|
"possible future skill",
|
|
"outline for a possible future skill"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.308,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"document package",
|
|
"wiki"
|
|
],
|
|
"no_build_directive": [
|
|
"not an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
|
|
"family": "long_context_near_neighbor",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0,
|
|
"lexical_support": 0.375,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow",
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"template only"
|
|
],
|
|
"no_build_directive": [
|
|
"no skill package",
|
|
"discussion only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_dev_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 8,
|
|
"passed": 8,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 7,
|
|
"passed": 7,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 7,
|
|
"passed": 7,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"workflow_to_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"iterate_existing_skill": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"complex_multi_asset": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"paraphrase_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"translate_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_not_package": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"partial_scaffold_not_full_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"document_export_vs_agent_skill": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"future_outline_vs_build": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_near_neighbor": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Convert this operations checklist into a reusable skill.",
|
|
"family": "workflow_to_skill",
|
|
"score": 0.379,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"checklist",
|
|
"operations checklist"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Add trigger evals to this skill before sharing it with the team.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.333,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.286,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"evals",
|
|
"trigger evals"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"this skill",
|
|
"sharing it with the team",
|
|
"before sharing it with the team"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"sharing it with the team"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.816,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.208,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package",
|
|
"turn all of that into one reusable skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"evals",
|
|
"packaging checks"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package",
|
|
"packaging",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook",
|
|
"process",
|
|
"prompt history"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.454,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.411,
|
|
"support_score": 0,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"production-ready"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow",
|
|
"workflow fragments",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.514,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.464,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.214,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"trigger boundary"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"existing skill"
|
|
],
|
|
"reuse_package": [
|
|
"distribution"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"distribution"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn the SOP, transcript, and cleanup notes into a maintained skill for the team library.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.39,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"team library",
|
|
"maintained"
|
|
],
|
|
"team_operationalize": [
|
|
"team"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Codify this workflow into a shared capability package and validate the route boundary before distribution.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.815,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.067,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"codify this workflow"
|
|
],
|
|
"eval_optimize": [
|
|
"validate",
|
|
"route boundary"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"distribution",
|
|
"shared capability"
|
|
],
|
|
"team_operationalize": [
|
|
"shared",
|
|
"distribution"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "After the long handoff notes below, build a skill package for team reuse. The materials include a transcript, rough notes, and an onboarding flow, and the result should include references, scripts, and validation.",
|
|
"family": "long_context_trigger",
|
|
"score": 0.871,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.857,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.154,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill",
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"validation"
|
|
],
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reuse",
|
|
"package",
|
|
"team reuse"
|
|
],
|
|
"team_operationalize": [
|
|
"team",
|
|
"team reuse"
|
|
],
|
|
"transform_workflow": [
|
|
"rough notes",
|
|
"transcript",
|
|
"onboarding flow",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Give me ideas for improving our process.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"ideas"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Review this note and tell me what it means in plain English.",
|
|
"family": "explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"tell me what it means",
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.232,
|
|
"support_score": 0,
|
|
"lexical_support": 0.133,
|
|
"negative_penalty": 0.66,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"reuse_package": [
|
|
"packaging"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm",
|
|
"improve packaging"
|
|
],
|
|
"no_build_directive": [
|
|
"do not generate any skill files"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Translate these notes into French and keep the structure exactly as-is.",
|
|
"family": "translate_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.35,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"translate_only": [
|
|
"translate",
|
|
"into french"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Summarize this workflow and list the main points only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"summarize_only": [
|
|
"summarize"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "This is a long incident write-up. Keep it as documentation only, explain it in simpler language, and do not convert it into a reusable capability.",
|
|
"family": "long_context_document_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain it",
|
|
"in simpler language"
|
|
],
|
|
"no_build_directive": [
|
|
"do not convert it into a reusable capability",
|
|
"keep it as documentation only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "I pasted a very long process note. For now I only need a recap and a short summary, not a skill package.",
|
|
"family": "long_context_summary_only",
|
|
"score": 0.236,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"summarize_only": [
|
|
"summary",
|
|
"recap"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Review this process note and explain it, no packaging needed.",
|
|
"family": "explain_not_package",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain it"
|
|
],
|
|
"no_build_directive": [
|
|
"no packaging needed"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this into a checklist and template, but stop short of making a full skill.",
|
|
"family": "partial_scaffold_not_full_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0.42,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"stop short of making a full skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Package this explanation as a document, not as an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.3,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document"
|
|
],
|
|
"no_build_directive": [
|
|
"not as an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Make a future-ready outline for this skill idea, but do not build the package.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"iterate_existing_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.321,
|
|
"support_score": 0,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"iterate_existing_skill": [
|
|
"this skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future-ready outline"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"reusable-looking document"
|
|
],
|
|
"no_build_directive": [
|
|
"without turning them into an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn the workflow below into a wiki-ready reference document and template only. We may build a skill later, but not now.",
|
|
"family": "long_context_near_neighbor",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.048,
|
|
"negative_penalty": 0.82,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"wiki",
|
|
"reference document",
|
|
"template only"
|
|
],
|
|
"future_outline": [
|
|
"not now",
|
|
"later but not now"
|
|
],
|
|
"no_build_directive": [
|
|
"not now",
|
|
"later but not now"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Shape this idea for later and produce a future-ready outline, but do not build the skill package yet.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.158,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later",
|
|
"future-ready outline"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 5,
|
|
"passed": 5,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 4,
|
|
"passed": 4,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 4,
|
|
"passed": 4,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"complex_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"iterate_existing_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"workflow_to_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"paraphrase_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_trigger": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"translate_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"future_outline_vs_build": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"document_export_vs_agent_skill": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"long_context_near_neighbor": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
|
|
"family": "complex_multi_asset",
|
|
"score": 0.465,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.411,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.211,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"team_operationalize": [
|
|
"internal"
|
|
],
|
|
"transform_workflow": [
|
|
"process",
|
|
"internal process note",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
|
|
"family": "iterate_existing_skill",
|
|
"score": 0.684,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.643,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.278,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"eval_optimize": [
|
|
"evals",
|
|
"near-neighbor",
|
|
"trigger boundary"
|
|
],
|
|
"iterate_existing_skill": [
|
|
"existing skill"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"distribution"
|
|
],
|
|
"team_operationalize": [
|
|
"distribution"
|
|
],
|
|
"transform_workflow": [
|
|
"existing skill draft"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
|
|
"family": "workflow_to_skill",
|
|
"score": 0.805,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill library entry"
|
|
],
|
|
"eval_optimize": [
|
|
"validation"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"library entry"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Codify this quarterly release routine into a reusable skill package with scripts, references, and packaging checks.",
|
|
"family": "paraphrase_trigger",
|
|
"score": 0.874,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.857,
|
|
"support_score": 0,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"packaging checks"
|
|
],
|
|
"multi_asset": [
|
|
"references",
|
|
"scripts"
|
|
],
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package",
|
|
"packaging",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"quarterly release routine",
|
|
"scripts"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Below is a noisy transcript plus release notes and prompt history. Ignore the chatter and turn this playbook into an agent capability package for the team library.",
|
|
"family": "long_context_trigger",
|
|
"score": 0.691,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"team_operationalize"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.16,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"turn this playbook into an agent capability package",
|
|
"agent capability"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"team library",
|
|
"release"
|
|
],
|
|
"team_operationalize": [
|
|
"team"
|
|
],
|
|
"transform_workflow": [
|
|
"prompt history",
|
|
"playbook",
|
|
"transcript",
|
|
"release notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.72,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"not a reusable skill"
|
|
],
|
|
"summarize_only": [
|
|
"summary",
|
|
"only want a summary"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain this release checklist in simpler language; do not package it.",
|
|
"family": "explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.182,
|
|
"negative_penalty": 0.68,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"package",
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain this",
|
|
"in simpler language"
|
|
],
|
|
"no_build_directive": [
|
|
"do not package it"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Keep this release workflow as documentation only for the wiki and produce a reference document, not for agent execution.",
|
|
"family": "long_context_document_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.167,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"release"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"wiki",
|
|
"reference document"
|
|
],
|
|
"no_build_directive": [
|
|
"not for agent execution"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Translate this long runbook into Russian and keep it as documentation only with no skill package.",
|
|
"family": "translate_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"no_build_directive",
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0,
|
|
"lexical_support": 0.188,
|
|
"negative_penalty": 0.77,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook"
|
|
]
|
|
},
|
|
"negative": {
|
|
"no_build_directive": [
|
|
"no skill package",
|
|
"keep it as documentation only"
|
|
],
|
|
"translate_only": [
|
|
"translate",
|
|
"into russian"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Help me shape an idea before we decide whether to build a skill.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.304,
|
|
"support_score": 0,
|
|
"lexical_support": 0.077,
|
|
"negative_penalty": 0.46,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"build a skill"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"shape an idea"
|
|
],
|
|
"future_outline": [
|
|
"before we decide whether to build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create an outline for a possible future skill, but do not build the skill yet.",
|
|
"family": "future_outline_vs_build",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.214,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill",
|
|
"possible future skill",
|
|
"outline for a possible future skill"
|
|
],
|
|
"no_build_directive": [
|
|
"do not build"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Turn this into a document package for the wiki, not an agent skill.",
|
|
"family": "document_export_vs_agent_skill",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.482,
|
|
"support_score": 0,
|
|
"lexical_support": 0.308,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"agent skill"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"document",
|
|
"document package",
|
|
"wiki"
|
|
],
|
|
"no_build_directive": [
|
|
"not an agent skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "From the long workflow below, create a template only and a checklist for discussion only. No skill package.",
|
|
"family": "long_context_near_neighbor",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.661,
|
|
"support_score": 0,
|
|
"lexical_support": 0.375,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow",
|
|
"checklist"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"template only"
|
|
],
|
|
"no_build_directive": [
|
|
"no skill package",
|
|
"discussion only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"winner_blind_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_future_outline": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
|
|
"family": "blind_operationalize",
|
|
"score": 0.38,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"support escalation routine"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
|
|
"family": "blind_multi_asset",
|
|
"score": 0.374,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
|
|
"family": "blind_document_only",
|
|
"score": 0.084,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only"
|
|
],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.232,
|
|
"support_score": 0,
|
|
"lexical_support": 0.136,
|
|
"negative_penalty": 0.18,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain the differences between prompts and skills in plain English.",
|
|
"family": "blind_explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
|
|
"family": "blind_future_outline",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.054,
|
|
"support_score": 0,
|
|
"lexical_support": 0.154,
|
|
"negative_penalty": 0.22,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
|
|
"family": "blind_brainstorm_only",
|
|
"score": 0.013,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.182,
|
|
"negative_penalty": 0.22,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_blind_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_future_outline": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
|
|
"family": "blind_operationalize",
|
|
"score": 0.38,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"support escalation routine"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
|
|
"family": "blind_multi_asset",
|
|
"score": 0.374,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
|
|
"family": "blind_document_only",
|
|
"score": 0.084,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only"
|
|
],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.232,
|
|
"support_score": 0,
|
|
"lexical_support": 0.136,
|
|
"negative_penalty": 0.18,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain the differences between prompts and skills in plain English.",
|
|
"family": "blind_explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
|
|
"family": "blind_future_outline",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.054,
|
|
"support_score": 0,
|
|
"lexical_support": 0.154,
|
|
"negative_penalty": 0.22,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
|
|
"family": "blind_brainstorm_only",
|
|
"score": 0.013,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.357,
|
|
"support_score": 0,
|
|
"lexical_support": 0.182,
|
|
"negative_penalty": 0.22,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"baseline_blind_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_future_outline": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
|
|
"family": "blind_operationalize",
|
|
"score": 0.393,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.071,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"support escalation routine"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
|
|
"family": "blind_multi_asset",
|
|
"score": 0.392,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
|
|
"family": "blind_document_only",
|
|
"score": 0.019,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"document_only"
|
|
],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0.06,
|
|
"lexical_support": 0.136,
|
|
"negative_penalty": 0.18,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain the differences between prompts and skills in plain English.",
|
|
"family": "blind_explain_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
|
|
"family": "blind_future_outline",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.06,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.22,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
|
|
"family": "blind_brainstorm_only",
|
|
"score": 0.024,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0.22,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"winner_judge_blind_holdout_report": {
|
|
"judge": "rubric-blind-v1",
|
|
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_future_outline": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
|
|
"family": "blind_operationalize",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 3,
|
|
"negative_vote": 0,
|
|
"margin": 3,
|
|
"confidence": 0.74,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"support escalation routine"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
|
|
"family": "blind_multi_asset",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"notes"
|
|
],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 4,
|
|
"negative_vote": 0,
|
|
"margin": 4,
|
|
"confidence": 0.82,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
|
|
"family": "blind_document_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"multi_asset"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 3,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [
|
|
"document_only"
|
|
],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 2,
|
|
"negative_vote": 1,
|
|
"margin": 1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Explain the differences between prompts and skills in plain English.",
|
|
"family": "blind_explain_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"prompts"
|
|
],
|
|
"artifact_hits": [
|
|
"prompts"
|
|
],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"explain",
|
|
"plain english"
|
|
],
|
|
"positive_vote": 2,
|
|
"negative_vote": 3,
|
|
"margin": -1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
|
|
"family": "blind_future_outline",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"notes"
|
|
],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 0,
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 1,
|
|
"negative_vote": 2,
|
|
"margin": -1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
|
|
"family": "blind_brainstorm_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 3,
|
|
"negative_vote": 2,
|
|
"margin": 1,
|
|
"confidence": 0.66,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
]
|
|
},
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"current_judge_blind_holdout_report": {
|
|
"judge": "rubric-blind-v1",
|
|
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_future_outline": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
|
|
"family": "blind_operationalize",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 3,
|
|
"negative_vote": 0,
|
|
"margin": 3,
|
|
"confidence": 0.74,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"support escalation routine"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
|
|
"family": "blind_multi_asset",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"notes"
|
|
],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 4,
|
|
"negative_vote": 0,
|
|
"margin": 4,
|
|
"confidence": 0.82,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
|
|
"family": "blind_document_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"multi_asset"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 3,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [
|
|
"document_only"
|
|
],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 2,
|
|
"negative_vote": 1,
|
|
"margin": 1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Explain the differences between prompts and skills in plain English.",
|
|
"family": "blind_explain_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"prompts"
|
|
],
|
|
"artifact_hits": [
|
|
"prompts"
|
|
],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"explain",
|
|
"plain english"
|
|
],
|
|
"positive_vote": 2,
|
|
"negative_vote": 3,
|
|
"margin": -1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
|
|
"family": "blind_future_outline",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"notes"
|
|
],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 0,
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 1,
|
|
"negative_vote": 2,
|
|
"margin": -1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
|
|
"family": "blind_brainstorm_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 3,
|
|
"negative_vote": 2,
|
|
"margin": 1,
|
|
"confidence": 0.66,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
]
|
|
},
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"baseline_judge_blind_holdout_report": {
|
|
"judge": "rubric-blind-v1",
|
|
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_multi_asset": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_document_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_future_outline": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Operationalize this support escalation routine as a maintained skill and keep trigger eval coverage.",
|
|
"family": "blind_operationalize",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 3,
|
|
"negative_vote": 0,
|
|
"margin": 3,
|
|
"confidence": 0.74,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"support escalation routine"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Take these SOP scraps, prompt notes, and transcript excerpts and formalize them into a reusable capability package.",
|
|
"family": "blind_multi_asset",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"notes"
|
|
],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 4,
|
|
"negative_vote": 0,
|
|
"margin": 4,
|
|
"confidence": 0.82,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"reusable",
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"sop",
|
|
"transcript"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Turn this workflow note into a wiki reference only, do not create any skill files, and keep it out of agent execution.",
|
|
"family": "blind_document_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 3,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [
|
|
"document_only"
|
|
],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 2,
|
|
"negative_vote": 1,
|
|
"margin": 1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Explain the differences between prompts and skills in plain English.",
|
|
"family": "blind_explain_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"prompts"
|
|
],
|
|
"artifact_hits": [
|
|
"prompts"
|
|
],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"explain",
|
|
"plain english"
|
|
],
|
|
"positive_vote": 2,
|
|
"negative_vote": 3,
|
|
"margin": -1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"explain_only": [
|
|
"plain english"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future skill outline from these notes, but stop before generating files.",
|
|
"family": "blind_future_outline",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [
|
|
"notes"
|
|
],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 0,
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 1,
|
|
"negative_vote": 2,
|
|
"margin": -1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Review this process and suggest improvements for later; no packaging yet.",
|
|
"family": "blind_brainstorm_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"transform_workflow",
|
|
"reuse_package"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"future_outline"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 3,
|
|
"negative_vote": 2,
|
|
"margin": 1,
|
|
"confidence": 0.66,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
],
|
|
"transform_workflow": [
|
|
"process"
|
|
]
|
|
},
|
|
"negative": {
|
|
"future_outline": [
|
|
"for later"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
]
|
|
},
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"winner_adversarial_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"adversarial_noisy_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_multi_asset_boundary": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_document_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_summary_translation_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_future_outline_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_brainstorm_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
|
|
"family": "adversarial_noisy_operationalize",
|
|
"score": 0.812,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.158,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"hardening"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
|
|
"family": "adversarial_multi_asset_boundary",
|
|
"score": 0.808,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.105,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"codify this routine"
|
|
],
|
|
"eval_optimize": [
|
|
"stress test",
|
|
"route boundary"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"prompt history"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
|
|
"family": "adversarial_document_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.411,
|
|
"support_score": 0,
|
|
"lexical_support": 0.286,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
],
|
|
"no_build_directive": [
|
|
"do not generate any skill files"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
|
|
"family": "adversarial_summary_translation_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive",
|
|
"summarize_only",
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only",
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 1.25,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"sop"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"knowledge base"
|
|
],
|
|
"no_build_directive": [
|
|
"keep it as documentation only"
|
|
],
|
|
"summarize_only": [
|
|
"summarize"
|
|
],
|
|
"translate_only": [
|
|
"translate",
|
|
"into japanese"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
|
|
"family": "adversarial_future_outline_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill",
|
|
"possible future skill"
|
|
],
|
|
"no_build_directive": [
|
|
"stop short of making a full skill",
|
|
"plain note",
|
|
"keep it as a plain note"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
|
|
"family": "adversarial_brainstorm_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0.88,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
],
|
|
"future_outline": [
|
|
"for later"
|
|
],
|
|
"no_build_directive": [
|
|
"discussion only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_adversarial_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"adversarial_noisy_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_multi_asset_boundary": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_document_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_summary_translation_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_future_outline_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_brainstorm_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
|
|
"family": "adversarial_noisy_operationalize",
|
|
"score": 0.812,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.158,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"hardening"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
|
|
"family": "adversarial_multi_asset_boundary",
|
|
"score": 0.808,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.804,
|
|
"support_score": 0,
|
|
"lexical_support": 0.105,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"codify this routine"
|
|
],
|
|
"eval_optimize": [
|
|
"stress test",
|
|
"route boundary"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"prompt history"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
|
|
"family": "adversarial_document_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"multi_asset",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.411,
|
|
"support_score": 0,
|
|
"lexical_support": 0.286,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
],
|
|
"no_build_directive": [
|
|
"do not generate any skill files"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
|
|
"family": "adversarial_summary_translation_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive",
|
|
"summarize_only",
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only",
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 1.25,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"sop"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"knowledge base"
|
|
],
|
|
"no_build_directive": [
|
|
"keep it as documentation only"
|
|
],
|
|
"summarize_only": [
|
|
"summarize"
|
|
],
|
|
"translate_only": [
|
|
"translate",
|
|
"into japanese"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
|
|
"family": "adversarial_future_outline_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill",
|
|
"possible future skill"
|
|
],
|
|
"no_build_directive": [
|
|
"stop short of making a full skill",
|
|
"plain note",
|
|
"keep it as a plain note"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
|
|
"family": "adversarial_brainstorm_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill",
|
|
"multi_asset"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.179,
|
|
"support_score": 0,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0.88,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
],
|
|
"future_outline": [
|
|
"for later"
|
|
],
|
|
"no_build_directive": [
|
|
"discussion only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"baseline_adversarial_holdout_report": {
|
|
"threshold": 0.33,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"adversarial_noisy_operationalize": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_multi_asset_boundary": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_document_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_summary_translation_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_future_outline_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_brainstorm_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.",
|
|
"family": "adversarial_noisy_operationalize",
|
|
"score": 0.845,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.849,
|
|
"support_score": 0,
|
|
"lexical_support": 0.053,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"skill package"
|
|
],
|
|
"eval_optimize": [
|
|
"hardening"
|
|
],
|
|
"reuse_package": [
|
|
"package",
|
|
"maintained",
|
|
"operationalize"
|
|
],
|
|
"transform_workflow": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.",
|
|
"family": "adversarial_multi_asset_boundary",
|
|
"score": 0.845,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"build_skill",
|
|
"eval_optimize",
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.849,
|
|
"support_score": 0,
|
|
"lexical_support": 0.053,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"build_skill": [
|
|
"codify this routine"
|
|
],
|
|
"eval_optimize": [
|
|
"stress test",
|
|
"route boundary"
|
|
],
|
|
"reuse_package": [
|
|
"reusable"
|
|
],
|
|
"transform_workflow": [
|
|
"prompt history"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Create a wiki package from this runbook and do not generate any skill files.",
|
|
"family": "adversarial_document_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package",
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"multi_asset"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.377,
|
|
"support_score": 0.06,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.6,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"multi_asset": [
|
|
"files"
|
|
],
|
|
"reuse_package": [
|
|
"package"
|
|
],
|
|
"transform_workflow": [
|
|
"runbook"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"wiki"
|
|
],
|
|
"no_build_directive": [
|
|
"do not generate any skill files"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.",
|
|
"family": "adversarial_summary_translation_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"transform_workflow"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"document_only",
|
|
"no_build_directive",
|
|
"summarize_only",
|
|
"translate_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"no_build_directive",
|
|
"summarize_only",
|
|
"translate_only"
|
|
],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0,
|
|
"lexical_support": 0.062,
|
|
"negative_penalty": 1.25,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"transform_workflow": [
|
|
"sop"
|
|
]
|
|
},
|
|
"negative": {
|
|
"document_only": [
|
|
"knowledge base"
|
|
],
|
|
"no_build_directive": [
|
|
"keep it as documentation only"
|
|
],
|
|
"summarize_only": [
|
|
"summarize"
|
|
],
|
|
"translate_only": [
|
|
"translate",
|
|
"into japanese"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.",
|
|
"family": "adversarial_future_outline_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.05,
|
|
"negative_penalty": 0.64,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"future_outline": [
|
|
"future skill",
|
|
"possible future skill"
|
|
],
|
|
"no_build_directive": [
|
|
"stop short of making a full skill",
|
|
"plain note",
|
|
"keep it as a plain note"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm packaging improvements for later and keep this as discussion only.",
|
|
"family": "adversarial_brainstorm_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"build_skill",
|
|
"transform_workflow",
|
|
"reuse_package",
|
|
"eval_optimize",
|
|
"iterate_existing_skill"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reuse_package"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only",
|
|
"future_outline",
|
|
"no_build_directive"
|
|
],
|
|
"semantic_coverage": 0.189,
|
|
"support_score": 0,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0.88,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reuse_package": [
|
|
"packaging"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
],
|
|
"future_outline": [
|
|
"for later"
|
|
],
|
|
"no_build_directive": [
|
|
"discussion only"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"candidates": [
|
|
{
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Create/improve/evaluate agent skills from workflows, prompts, SOPs, scripts. Use for migration/release/package, routing, evals/tests, install checks, 优化已有 skill, 补 trigger 评测. Exclude no-skill summary/translation/docs.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 54,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
54
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "minimal",
|
|
"label": "Minimal",
|
|
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
|
|
"strategy": "minimal_template",
|
|
"estimated_tokens": 41,
|
|
"dev": {
|
|
"false_positives": 1,
|
|
"false_negatives": 1,
|
|
"precision": 0.875,
|
|
"recall": 0.875,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 0.857
|
|
},
|
|
"selection_key": [
|
|
1,
|
|
1,
|
|
-1.0,
|
|
-0.857,
|
|
-0.875,
|
|
-0.875,
|
|
41
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "guardrail",
|
|
"label": "Guardrail",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
|
|
"strategy": "guardrail_template",
|
|
"estimated_tokens": 56,
|
|
"dev": {
|
|
"false_positives": 1,
|
|
"false_negatives": 1,
|
|
"precision": 0.875,
|
|
"recall": 0.875,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 0.857
|
|
},
|
|
"selection_key": [
|
|
1,
|
|
1,
|
|
-1.0,
|
|
-0.857,
|
|
-0.875,
|
|
-0.875,
|
|
56
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "balanced",
|
|
"label": "Balanced",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
|
|
"strategy": "balanced_template",
|
|
"estimated_tokens": 60,
|
|
"dev": {
|
|
"false_positives": 1,
|
|
"false_negatives": 1,
|
|
"precision": 0.875,
|
|
"recall": 0.875,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 0.857
|
|
},
|
|
"selection_key": [
|
|
1,
|
|
1,
|
|
-1.0,
|
|
-0.857,
|
|
-0.875,
|
|
-0.875,
|
|
60
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "artifact_aware",
|
|
"label": "Artifact Aware",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
|
|
"strategy": "artifact_template",
|
|
"estimated_tokens": 77,
|
|
"dev": {
|
|
"false_positives": 1,
|
|
"false_negatives": 1,
|
|
"precision": 0.875,
|
|
"recall": 0.875,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 0.857
|
|
},
|
|
"selection_key": [
|
|
1,
|
|
1,
|
|
-1.0,
|
|
-0.857,
|
|
-0.875,
|
|
-0.875,
|
|
77
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "boundary",
|
|
"label": "Boundary",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
|
|
"strategy": "boundary_template",
|
|
"estimated_tokens": 83,
|
|
"dev": {
|
|
"false_positives": 1,
|
|
"false_negatives": 1,
|
|
"precision": 0.875,
|
|
"recall": 0.875,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 0.857
|
|
},
|
|
"selection_key": [
|
|
1,
|
|
1,
|
|
-1.0,
|
|
-0.857,
|
|
-0.875,
|
|
-0.875,
|
|
83
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
}
|
|
],
|
|
"selection_logic": {
|
|
"priority": [
|
|
"fewest false positives",
|
|
"fewest false negatives",
|
|
"highest near-neighbor pass rate",
|
|
"highest negative pass rate",
|
|
"highest precision",
|
|
"highest recall",
|
|
"shortest description"
|
|
]
|
|
},
|
|
"comparison": {
|
|
"winner_vs_current_dev": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_current_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_current_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_dev": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 1,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": -1,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 0.875,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_current_judge_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_judge_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_current_adversarial_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_adversarial_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
}
|
|
},
|
|
"acceptance_gates": {
|
|
"selection_basis": "dev only",
|
|
"holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"winner_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.704,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.465,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.465,
|
|
"threshold_margin": 0.135,
|
|
"positive_threshold_buffer": 0.135,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"current_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.704,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.465,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.465,
|
|
"threshold_margin": 0.135,
|
|
"positive_threshold_buffer": 0.135,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"baseline_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.707,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.416,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.416,
|
|
"threshold_margin": 0.086,
|
|
"positive_threshold_buffer": 0.086,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"winner_family_health": {
|
|
"family_count": 12,
|
|
"clean_family_count": 12,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "workflow_to_skill",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 12,
|
|
"clean_family_count": 12,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "workflow_to_skill",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 12,
|
|
"clean_family_count": 12,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "workflow_to_skill",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"blind_holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"winner_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.377,
|
|
"mean_non_trigger_score": 0.024,
|
|
"mean_near_neighbor_score": 0.006,
|
|
"min_positive_score": 0.374,
|
|
"max_non_trigger_score": 0.084,
|
|
"score_gap": 0.29,
|
|
"threshold_margin": 0.044,
|
|
"positive_threshold_buffer": 0.044,
|
|
"negative_threshold_buffer": 0.246,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "watch"
|
|
},
|
|
"current_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.377,
|
|
"mean_non_trigger_score": 0.024,
|
|
"mean_near_neighbor_score": 0.006,
|
|
"min_positive_score": 0.374,
|
|
"max_non_trigger_score": 0.084,
|
|
"score_gap": 0.29,
|
|
"threshold_margin": 0.044,
|
|
"positive_threshold_buffer": 0.044,
|
|
"negative_threshold_buffer": 0.246,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "watch"
|
|
},
|
|
"baseline_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.393,
|
|
"mean_non_trigger_score": 0.011,
|
|
"mean_near_neighbor_score": 0.012,
|
|
"min_positive_score": 0.392,
|
|
"max_non_trigger_score": 0.024,
|
|
"score_gap": 0.368,
|
|
"threshold_margin": 0.062,
|
|
"positive_threshold_buffer": 0.062,
|
|
"negative_threshold_buffer": 0.306,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "watch"
|
|
},
|
|
"winner_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"judge_blind_holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0,
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0,
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0,
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"winner_calibration": null,
|
|
"current_calibration": null,
|
|
"baseline_calibration": null,
|
|
"winner_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"adversarial_holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"winner_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.81,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.808,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.808,
|
|
"threshold_margin": 0.33,
|
|
"positive_threshold_buffer": 0.478,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"current_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.81,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.808,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.808,
|
|
"threshold_margin": 0.33,
|
|
"positive_threshold_buffer": 0.478,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"baseline_calibration": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.845,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.845,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.845,
|
|
"threshold_margin": 0.33,
|
|
"positive_threshold_buffer": 0.515,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"winner_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_summary_translation_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_summary_translation_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_summary_translation_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"summary": {
|
|
"winner_label": "Current",
|
|
"winner_tokens": 54,
|
|
"current_tokens": 54,
|
|
"winner_dev_total_errors": 0,
|
|
"current_dev_total_errors": 0,
|
|
"winner_holdout_total_errors": 0,
|
|
"current_holdout_total_errors": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"current_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"current_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"current_adversarial_holdout_total_errors": 0,
|
|
"winner_judge_blind_agreement_rate": 1.0,
|
|
"winner_adversarial_risk_band": "healthy",
|
|
"winner_adversarial_score_gap": 0.808,
|
|
"candidate_count": 6,
|
|
"baseline_tokens": 8,
|
|
"baseline_dev_total_errors": 1,
|
|
"baseline_holdout_total_errors": 0,
|
|
"baseline_blind_holdout_total_errors": 0,
|
|
"baseline_judge_blind_holdout_total_errors": 0,
|
|
"baseline_adversarial_holdout_total_errors": 0
|
|
}
|
|
}
|