frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
98d2593647129e493dcefcadea286c4dcf77e51e
yao-meta-skill/examples/governed-incident-command/optimization/reports/description_optimization.json
T
yaojingang 98d2593647 feat: add artifact design quality profiles
2026-05-01 14:27:51 +08:00

6065 lines
186 KiB
JSON
Raw Blame History

{
"current_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"current_candidate": {
"id": "current",
"label": "Current",
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"strategy": "current",
"estimated_tokens": 37,
"dev": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
1,
-1.0,
-1.0,
-1.0,
-0.667,
37
],
"holdout": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
"baseline": {
"description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.",
"estimated_tokens": 93,
"dev": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"bucket_stats": {
"should_trigger": {
"total": 3,
"passed": 2,
"pass_rate": 0.667
},
"should_not_trigger": {
"total": 3,
"passed": 3,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"packet_assembly": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"governed_workflow": {
"total": 2,
"passed": 1,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.5
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"generic_debugging": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "governed_workflow",
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
"score": 0.392,
"reason": "false_negative",
"matched_desired_concepts": [
"governed_workflow",
"reusable_packaging",
"severity_assessment"
],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
"family": "packet_assembly",
"score": 0.686,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"governed_workflow",
"incident_inputs",
"incident_packet"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.62,
"support_score": 0,
"lexical_support": 0.692,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"incident notes",
"alerts",
"transcripts"
],
"incident_packet": [
"incident command packet"
]
},
"negative": {}
}
}
},
{
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
"family": "governed_workflow",
"score": 0.791,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"incident_inputs",
"incident_packet",
"reusable_packaging",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.74,
"support_score": 0,
"lexical_support": 0.625,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"incident_inputs": [
"outage workflow"
],
"incident_packet": [
"incident review skill"
],
"reusable_packaging": [
"reusable",
"workflow",
"skill"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
"family": "governed_workflow",
"score": 0.392,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"governed_workflow",
"reusable_packaging",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.32,
"support_score": 0,
"lexical_support": 0.467,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"governed_workflow": [
"maintained",
"operational skill"
],
"reusable_packaging": [
"skill",
"package"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Explain what incident severity means.",
"family": "explain_only",
"score": 0.198,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.18,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Draft one short outage update for customers.",
"family": "one_off_update",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"one_off_update"
],
"exclusive_negative_concepts": [
"one_off_update"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.286,
"negative_penalty": 0.34,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"one_off_update": [
"draft one short outage update"
]
}
}
}
},
{
"prompt": "Debug the service outage and find the root cause.",
"family": "generic_debugging",
"score": 0.01,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.125,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
],
"near_neighbor": [
{
"prompt": "Summarize this incident but do not package it as a reusable skill.",
"family": "summary_only",
"score": 0.07,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"reusable_packaging"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.417,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable",
"skill",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"governed_workflow",
"reusable_packaging"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.14,
"support_score": 0,
"lexical_support": 0.429,
"negative_penalty": 0.24,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"reusable_packaging": [
"workflow"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
}
},
"holdout": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 2,
"precision": null,
"recall": 0.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 0,
"pass_rate": 0.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"governed_workflow": {
"total": 1,
"passed": 0,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
},
"packet_assembly": {
"total": 1,
"passed": 0,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "governed_workflow",
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"score": 0.399,
"reason": "false_negative",
"matched_desired_concepts": [
"governed_workflow",
"reusable_packaging",
"severity_assessment"
],
"matched_negative_concepts": []
},
{
"bucket": "should_trigger",
"family": "packet_assembly",
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
"score": 0.321,
"reason": "false_negative",
"matched_desired_concepts": [
"incident_inputs",
"reusable_packaging"
],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"family": "governed_workflow",
"score": 0.399,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"governed_workflow",
"reusable_packaging",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.32,
"support_score": 0,
"lexical_support": 0.562,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"reusable_packaging": [
"workflow"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
"family": "packet_assembly",
"score": 0.321,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"incident_inputs",
"reusable_packaging"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.24,
"support_score": 0,
"lexical_support": 0.75,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"incident_inputs": [
"alerts"
],
"reusable_packaging": [
"reusable",
"skill"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Write a single executive summary of this outage only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
},
{
"prompt": "Brainstorm what the team could do better next time.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.111,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Draft a one-off internal status update from these incident notes.",
"family": "one_off_update",
"score": 0.22,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"incident_inputs"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.2,
"support_score": 0,
"lexical_support": 0.455,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"incident notes"
]
},
"negative": {}
}
}
},
{
"prompt": "Explain how incident command works at a high level.",
"family": "explain_only",
"score": 0.018,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.222,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
]
}
}
},
"winner": {
"id": "current",
"label": "Current",
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"strategy": "current",
"estimated_tokens": 37,
"dev": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
1,
-1.0,
-1.0,
-1.0,
-0.667,
37
],
"holdout": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
"winner_dev_report": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"bucket_stats": {
"should_trigger": {
"total": 3,
"passed": 2,
"pass_rate": 0.667
},
"should_not_trigger": {
"total": 3,
"passed": 3,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"packet_assembly": {
"total": 1,
"passed": 0,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
},
"governed_workflow": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"generic_debugging": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "packet_assembly",
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
"score": 0.348,
"reason": "false_negative",
"matched_desired_concepts": [
"governed_workflow"
],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
"family": "packet_assembly",
"score": 0.348,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow"
],
"extra_positive_concepts": [
"incident_inputs",
"incident_packet"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.227,
"support_score": 0.52,
"lexical_support": 0.231,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"incident notes",
"alerts",
"transcripts"
],
"incident_packet": [
"incident command packet"
]
},
"negative": {}
}
}
},
{
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
"family": "governed_workflow",
"score": 0.521,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [
"incident_inputs",
"incident_packet",
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.409,
"support_score": 0.56,
"lexical_support": 0.312,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"outage workflow"
],
"incident_packet": [
"incident review skill"
],
"reusable_packaging": [
"reusable",
"workflow",
"skill"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
"family": "governed_workflow",
"score": 0.651,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.2,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"governed_workflow": [
"maintained",
"operational skill"
],
"reusable_packaging": [
"skill",
"package"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Explain what incident severity means.",
"family": "explain_only",
"score": 0.408,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.409,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Draft one short outage update for customers.",
"family": "one_off_update",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"one_off_update"
],
"exclusive_negative_concepts": [
"one_off_update"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.34,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"one_off_update": [
"draft one short outage update"
]
}
}
}
},
{
"prompt": "Debug the service outage and find the root cause.",
"family": "generic_debugging",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
],
"near_neighbor": [
{
"prompt": "Summarize this incident but do not package it as a reusable skill.",
"family": "summary_only",
"score": 0.017,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.04,
"lexical_support": 0.083,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable",
"skill",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.227,
"support_score": 0.04,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"reusable_packaging": [
"workflow"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
}
},
"winner_holdout_report": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 1,
"pass_rate": 0.5
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"governed_workflow": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"packet_assembly": {
"total": 1,
"passed": 0,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "packet_assembly",
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
"score": 0.08,
"reason": "false_negative",
"matched_desired_concepts": [],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"family": "governed_workflow",
"score": 0.66,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.312,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"reusable_packaging": [
"workflow"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
"family": "packet_assembly",
"score": 0.08,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"incident_inputs",
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.24,
"lexical_support": 0.25,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"alerts"
],
"reusable_packaging": [
"reusable",
"skill"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Write a single executive summary of this outage only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
},
{
"prompt": "Brainstorm what the team could do better next time.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Draft a one-off internal status update from these incident notes.",
"family": "one_off_update",
"score": 0.057,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"incident_inputs"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.2,
"lexical_support": 0.091,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"incident notes"
]
},
"negative": {}
}
}
},
{
"prompt": "Explain how incident command works at a high level.",
"family": "explain_only",
"score": 0.018,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.222,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
]
}
},
"current_dev_report": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"bucket_stats": {
"should_trigger": {
"total": 3,
"passed": 2,
"pass_rate": 0.667
},
"should_not_trigger": {
"total": 3,
"passed": 3,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"packet_assembly": {
"total": 1,
"passed": 0,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
},
"governed_workflow": {
"total": 2,
"passed": 2,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"generic_debugging": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "packet_assembly",
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
"score": 0.348,
"reason": "false_negative",
"matched_desired_concepts": [
"governed_workflow"
],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
"family": "packet_assembly",
"score": 0.348,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow"
],
"extra_positive_concepts": [
"incident_inputs",
"incident_packet"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.227,
"support_score": 0.52,
"lexical_support": 0.231,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"incident notes",
"alerts",
"transcripts"
],
"incident_packet": [
"incident command packet"
]
},
"negative": {}
}
}
},
{
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
"family": "governed_workflow",
"score": 0.521,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [
"incident_inputs",
"incident_packet",
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.409,
"support_score": 0.56,
"lexical_support": 0.312,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"outage workflow"
],
"incident_packet": [
"incident review skill"
],
"reusable_packaging": [
"reusable",
"workflow",
"skill"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
"family": "governed_workflow",
"score": 0.651,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.2,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"governed_workflow": [
"maintained",
"operational skill"
],
"reusable_packaging": [
"skill",
"package"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Explain what incident severity means.",
"family": "explain_only",
"score": 0.408,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.409,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Draft one short outage update for customers.",
"family": "one_off_update",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"one_off_update"
],
"exclusive_negative_concepts": [
"one_off_update"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.34,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"one_off_update": [
"draft one short outage update"
]
}
}
}
},
{
"prompt": "Debug the service outage and find the root cause.",
"family": "generic_debugging",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
],
"near_neighbor": [
{
"prompt": "Summarize this incident but do not package it as a reusable skill.",
"family": "summary_only",
"score": 0.017,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.04,
"lexical_support": 0.083,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable",
"skill",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.227,
"support_score": 0.04,
"lexical_support": 0.286,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"reusable_packaging": [
"workflow"
]
},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
}
},
"current_holdout_report": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 1,
"pass_rate": 0.5
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"governed_workflow": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"packet_assembly": {
"total": 1,
"passed": 0,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
},
"summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "packet_assembly",
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
"score": 0.08,
"reason": "false_negative",
"matched_desired_concepts": [],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
"family": "governed_workflow",
"score": 0.66,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.312,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"reusable_packaging": [
"workflow"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
"family": "packet_assembly",
"score": 0.08,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"incident_inputs",
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.24,
"lexical_support": 0.25,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"alerts"
],
"reusable_packaging": [
"reusable",
"skill"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Write a single executive summary of this outage only.",
"family": "summary_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
},
{
"prompt": "Brainstorm what the team could do better next time.",
"family": "brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.0,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Draft a one-off internal status update from these incident notes.",
"family": "one_off_update",
"score": 0.057,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"incident_inputs"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.2,
"lexical_support": 0.091,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"incident notes"
]
},
"negative": {}
}
}
},
{
"prompt": "Explain how incident command works at a high level.",
"family": "explain_only",
"score": 0.018,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.222,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
]
}
},
"winner_blind_holdout_report": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_packet_assembly": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_governed_workflow": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
"family": "blind_packet_assembly",
"score": 1.0,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"comms_and_owners",
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"incident_packet"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 1.0,
"support_score": 0.32,
"lexical_support": 0.176,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"score": 0.804,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"comms_and_owners",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.773,
"support_score": 0.04,
"lexical_support": 0.533,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"incident communication"
],
"reusable_packaging": [
"reusable",
"workflow"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Explain how severity levels are decided during incidents.",
"family": "blind_explain_only",
"score": 0.386,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.409,
"support_score": 0,
"lexical_support": 0.125,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Write a single incident update for customers from these notes.",
"family": "blind_one_off_update",
"score": 0.008,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.1,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
],
"near_neighbor": [
{
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
"family": "blind_summary_only",
"score": 0.276,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow"
],
"extra_positive_concepts": [
"incident_inputs",
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.227,
"support_score": 0.24,
"lexical_support": 0.091,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"timeline"
],
"reusable_packaging": [
"workflow",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
"family": "blind_brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
}
},
"current_blind_holdout_report": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_packet_assembly": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_governed_workflow": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
"family": "blind_packet_assembly",
"score": 1.0,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"comms_and_owners",
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"incident_packet"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 1.0,
"support_score": 0.32,
"lexical_support": 0.176,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"score": 0.804,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"comms_and_owners",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.773,
"support_score": 0.04,
"lexical_support": 0.533,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"incident communication"
],
"reusable_packaging": [
"reusable",
"workflow"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Explain how severity levels are decided during incidents.",
"family": "blind_explain_only",
"score": 0.386,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.409,
"support_score": 0,
"lexical_support": 0.125,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Write a single incident update for customers from these notes.",
"family": "blind_one_off_update",
"score": 0.008,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.1,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
],
"near_neighbor": [
{
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
"family": "blind_summary_only",
"score": 0.276,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow"
],
"extra_positive_concepts": [
"incident_inputs",
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.227,
"support_score": 0.24,
"lexical_support": 0.091,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"timeline"
],
"reusable_packaging": [
"workflow",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
"family": "blind_brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.083,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
}
},
"baseline_blind_holdout_report": {
"threshold": 0.48,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 1,
"pass_rate": 0.5
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_packet_assembly": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_governed_workflow": {
"total": 1,
"passed": 0,
"false_positives": 0,
"false_negatives": 1,
"pass_rate": 0.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [
{
"bucket": "should_trigger",
"family": "blind_governed_workflow",
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"score": 0.458,
"reason": "false_negative",
"matched_desired_concepts": [
"comms_and_owners",
"reusable_packaging",
"severity_assessment"
],
"matched_negative_concepts": []
}
],
"results": {
"should_trigger": [
{
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
"family": "blind_packet_assembly",
"score": 0.792,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"comms_and_owners",
"governed_workflow",
"incident_packet",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.76,
"support_score": 0,
"lexical_support": 0.412,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"score": 0.458,
"predicted_trigger": false,
"expected_trigger": true,
"passed": false,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"comms_and_owners",
"reusable_packaging",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.38,
"support_score": 0,
"lexical_support": 0.6,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"incident communication"
],
"reusable_packaging": [
"reusable",
"workflow"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Explain how severity levels are decided during incidents.",
"family": "blind_explain_only",
"score": 0.176,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.18,
"support_score": 0,
"lexical_support": 0.125,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Write a single incident update for customers from these notes.",
"family": "blind_one_off_update",
"score": 0.032,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {}
}
}
}
],
"near_neighbor": [
{
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
"family": "blind_summary_only",
"score": 0.395,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"governed_workflow",
"incident_inputs",
"reusable_packaging"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.34,
"support_score": 0,
"lexical_support": 0.273,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"timeline"
],
"reusable_packaging": [
"workflow",
"package"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
"family": "blind_brainstorm_only",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"brainstorm_only"
],
"exclusive_negative_concepts": [
"brainstorm_only"
],
"semantic_coverage": 0.0,
"support_score": 0,
"lexical_support": 0.167,
"negative_penalty": 0.24,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
}
},
"winner_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_packet_assembly": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_governed_workflow": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
"family": "blind_packet_assembly",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"strong_positive_concepts": [
"severity_assessment",
"comms_and_owners"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment",
"comms_and_owners"
],
"strong_positive_concepts": [
"severity_assessment",
"comms_and_owners"
],
"trigger_action_hits": [
"run severity assessment",
"assemble incident communication"
],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 5,
"negative_vote": 0,
"margin": 5,
"confidence": 0.96,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"incident communication"
],
"reusable_packaging": [
"reusable",
"workflow"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Explain how severity levels are decided during incidents.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment"
],
"strong_positive_concepts": [
"severity_assessment"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain"
],
"positive_vote": 1,
"negative_vote": 1,
"margin": 0,
"confidence": 0.5,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Write a single incident update for customers from these notes.",
"family": "blind_one_off_update",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 0,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
"family": "blind_summary_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"governed_workflow"
],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"summarize"
],
"positive_vote": 1,
"negative_vote": 1,
"margin": 0,
"confidence": 0.5,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"timeline"
],
"reusable_packaging": [
"workflow",
"package"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"brainstorm_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"brainstorm"
],
"positive_vote": 1,
"negative_vote": 3,
"margin": -2,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.657,
"rubric_version": "blind-v1"
}
},
"current_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_packet_assembly": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_governed_workflow": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
"family": "blind_packet_assembly",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"strong_positive_concepts": [
"severity_assessment",
"comms_and_owners"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment",
"comms_and_owners"
],
"strong_positive_concepts": [
"severity_assessment",
"comms_and_owners"
],
"trigger_action_hits": [
"run severity assessment",
"assemble incident communication"
],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 5,
"negative_vote": 0,
"margin": 5,
"confidence": 0.96,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"incident communication"
],
"reusable_packaging": [
"reusable",
"workflow"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Explain how severity levels are decided during incidents.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment"
],
"strong_positive_concepts": [
"severity_assessment"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain"
],
"positive_vote": 1,
"negative_vote": 1,
"margin": 0,
"confidence": 0.5,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Write a single incident update for customers from these notes.",
"family": "blind_one_off_update",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 0,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
"family": "blind_summary_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"governed_workflow"
],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"summarize"
],
"positive_vote": 1,
"negative_vote": 1,
"margin": 0,
"confidence": 0.5,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"timeline"
],
"reusable_packaging": [
"workflow",
"package"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"brainstorm_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"brainstorm"
],
"positive_vote": 1,
"negative_vote": 3,
"margin": -2,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.657,
"rubric_version": "blind-v1"
}
},
"baseline_judge_blind_holdout_report": {
"judge": "rubric-blind-v1",
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"blind_packet_assembly": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_governed_workflow": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_explain_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_one_off_update": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_summary_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"blind_brainstorm_only": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
"family": "blind_packet_assembly",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"incident_packet",
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"strong_positive_concepts": [
"incident_packet",
"severity_assessment",
"comms_and_owners"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 2,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 4,
"negative_vote": 0,
"margin": 4,
"confidence": 0.82,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
"family": "blind_governed_workflow",
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"reusable_packaging"
],
"strong_positive_concepts": [
"severity_assessment",
"comms_and_owners"
],
"trigger_action_hits": [
"run severity assessment",
"assemble incident communication"
],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 6,
"negative_vote": 0,
"margin": 6,
"confidence": 1.0,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"incident communication"
],
"reusable_packaging": [
"reusable",
"workflow"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Explain how severity levels are decided during incidents.",
"family": "blind_explain_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"severity_assessment"
],
"strong_positive_concepts": [
"severity_assessment"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 0,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"explain"
],
"positive_vote": 1,
"negative_vote": 1,
"margin": 0,
"confidence": 0.5,
"concept_evidence": {
"positive": {
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
},
{
"prompt": "Write a single incident update for customers from these notes.",
"family": "blind_one_off_update",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [],
"positive_vote": 1,
"negative_vote": 0,
"margin": 1,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {}
}
},
"boundary_case": true
}
],
"near_neighbor": [
{
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
"family": "blind_summary_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [
"incident_inputs",
"governed_workflow",
"reusable_packaging"
],
"strong_positive_concepts": [
"incident_inputs"
],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"summarize"
],
"positive_vote": 3,
"negative_vote": 1,
"margin": 2,
"confidence": 0.66,
"concept_evidence": {
"positive": {
"governed_workflow": [
"governed"
],
"incident_inputs": [
"timeline"
],
"reusable_packaging": [
"workflow",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
"family": "blind_brainstorm_only",
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"judge_detail": {
"mode": "judge-rubric",
"focused_positive_concepts": [],
"strong_positive_concepts": [],
"trigger_action_hits": [],
"input_hits": [],
"artifact_hits": [],
"capability_overlap": 1,
"exclusive_negative_concepts": [
"brainstorm_only"
],
"nonexclusive_negative_concepts": [],
"exclusion_hits": [],
"generic_exclusion_hits": [
"brainstorm"
],
"positive_vote": 1,
"negative_vote": 3,
"margin": -2,
"confidence": 0.58,
"concept_evidence": {
"positive": {},
"negative": {
"brainstorm_only": [
"brainstorm"
]
}
}
}
}
]
},
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.69,
"rubric_version": "blind-v1"
}
},
"winner_adversarial_holdout_report": {
"threshold": 0.34,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_governed_packet_from_noise": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_operational_skill_route": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_single_update_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_explain_and_summary_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_playbook_not_packet": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_debugging_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.",
"family": "adversarial_governed_packet_from_noise",
"score": 1.0,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"comms_and_owners",
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"incident_inputs",
"incident_packet"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 1.0,
"support_score": 0.52,
"lexical_support": 0.263,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_inputs": [
"alerts",
"transcripts",
"stakeholder requirements"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.",
"family": "adversarial_operational_skill_route",
"score": 0.654,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.235,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"governed_workflow": [
"maintained",
"operational skill"
],
"reusable_packaging": [
"skill",
"package"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Write one short customer outage update from these incident notes and stop there.",
"family": "adversarial_single_update_collision",
"score": 0.056,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"incident_inputs"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.2,
"lexical_support": 0.077,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"incident notes"
]
},
"negative": {}
}
}
},
{
"prompt": "Explain incident severity and summarize the outage timeline for training.",
"family": "adversarial_explain_and_summary_collision",
"score": 0.032,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [
"incident_inputs"
],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.409,
"support_score": 0.2,
"lexical_support": 0.2,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"timeline"
],
"severity_assessment": [
"severity"
]
},
"negative": {
"explain_only": [
"explain incident severity"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.",
"family": "adversarial_future_playbook_not_packet",
"score": 0.021,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.04,
"lexical_support": 0.143,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable",
"workflow",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.",
"family": "adversarial_debugging_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [
"generic_debugging"
],
"exclusive_negative_concepts": [
"generic_debugging"
],
"semantic_coverage": 0.0,
"support_score": 0.04,
"lexical_support": 0.154,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable"
]
},
"negative": {
"generic_debugging": [
"fix the service"
]
}
}
}
}
]
}
},
"current_adversarial_holdout_report": {
"threshold": 0.34,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_governed_packet_from_noise": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_operational_skill_route": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_single_update_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_explain_and_summary_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_playbook_not_packet": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_debugging_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.",
"family": "adversarial_governed_packet_from_noise",
"score": 1.0,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"comms_and_owners",
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"incident_inputs",
"incident_packet"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 1.0,
"support_score": 0.52,
"lexical_support": 0.263,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_inputs": [
"alerts",
"transcripts",
"stakeholder requirements"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.",
"family": "adversarial_operational_skill_route",
"score": 0.654,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"governed_workflow",
"severity_assessment"
],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.636,
"support_score": 0.04,
"lexical_support": 0.235,
"negative_penalty": 0,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"governed_workflow": [
"maintained",
"operational skill"
],
"reusable_packaging": [
"skill",
"package"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
}
}
],
"should_not_trigger": [
{
"prompt": "Write one short customer outage update from these incident notes and stop there.",
"family": "adversarial_single_update_collision",
"score": 0.056,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"incident_inputs"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.2,
"lexical_support": 0.077,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"incident notes"
]
},
"negative": {}
}
}
},
{
"prompt": "Explain incident severity and summarize the outage timeline for training.",
"family": "adversarial_explain_and_summary_collision",
"score": 0.032,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [
"severity_assessment"
],
"extra_positive_concepts": [
"incident_inputs"
],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.409,
"support_score": 0.2,
"lexical_support": 0.2,
"negative_penalty": 0.26,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"timeline"
],
"severity_assessment": [
"severity"
]
},
"negative": {
"explain_only": [
"explain incident severity"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.",
"family": "adversarial_future_playbook_not_packet",
"score": 0.021,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.0,
"support_score": 0.04,
"lexical_support": 0.143,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable",
"workflow",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.",
"family": "adversarial_debugging_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"severity_assessment",
"comms_and_owners",
"governed_workflow"
],
"matched_desired_concepts": [],
"extra_positive_concepts": [
"reusable_packaging"
],
"matched_negative_concepts": [
"generic_debugging"
],
"exclusive_negative_concepts": [
"generic_debugging"
],
"semantic_coverage": 0.0,
"support_score": 0.04,
"lexical_support": 0.154,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable"
]
},
"negative": {
"generic_debugging": [
"fix the service"
]
}
}
}
}
]
}
},
"baseline_adversarial_holdout_report": {
"threshold": 0.34,
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"bucket_stats": {
"should_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"should_not_trigger": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
},
"near_neighbor": {
"total": 2,
"passed": 2,
"pass_rate": 1.0
}
},
"family_stats": {
"adversarial_governed_packet_from_noise": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_operational_skill_route": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_single_update_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_explain_and_summary_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_future_playbook_not_packet": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
},
"adversarial_debugging_collision": {
"total": 1,
"passed": 1,
"false_positives": 0,
"false_negatives": 0,
"pass_rate": 1.0
}
},
"misfires": [],
"results": {
"should_trigger": [
{
"prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.",
"family": "adversarial_governed_packet_from_noise",
"score": 0.99,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"comms_and_owners",
"governed_workflow",
"incident_inputs",
"incident_packet",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.96,
"support_score": 0,
"lexical_support": 0.579,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"comms_and_owners": [
"owner"
],
"governed_workflow": [
"governed"
],
"incident_inputs": [
"alerts",
"transcripts",
"stakeholder requirements"
],
"incident_packet": [
"governed incident packet"
],
"severity_assessment": [
"severity assessment",
"severity"
]
},
"negative": {}
}
}
},
{
"prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.",
"family": "adversarial_operational_skill_route",
"score": 0.387,
"predicted_trigger": true,
"expected_trigger": true,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"governed_workflow",
"reusable_packaging",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.32,
"support_score": 0,
"lexical_support": 0.412,
"negative_penalty": 0,
"coverage_boost": 0.06,
"concept_evidence": {
"positive": {
"governed_workflow": [
"maintained",
"operational skill"
],
"reusable_packaging": [
"skill",
"package"
],
"severity_assessment": [
"severity"
]
},
"negative": {}
}
},
"boundary_case": true
}
],
"should_not_trigger": [
{
"prompt": "Write one short customer outage update from these incident notes and stop there.",
"family": "adversarial_single_update_collision",
"score": 0.215,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"incident_inputs"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.2,
"support_score": 0,
"lexical_support": 0.385,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"incident_inputs": [
"incident notes"
]
},
"negative": {}
}
}
},
{
"prompt": "Explain incident severity and summarize the outage timeline for training.",
"family": "adversarial_explain_and_summary_collision",
"score": 0.012,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"incident_inputs",
"severity_assessment"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"explain_only"
],
"exclusive_negative_concepts": [
"explain_only"
],
"semantic_coverage": 0.38,
"support_score": 0,
"lexical_support": 0.4,
"negative_penalty": 0.26,
"coverage_boost": 0.04,
"concept_evidence": {
"positive": {
"incident_inputs": [
"timeline"
],
"severity_assessment": [
"severity"
]
},
"negative": {
"explain_only": [
"explain incident severity"
]
}
}
}
}
],
"near_neighbor": [
{
"prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.",
"family": "adversarial_future_playbook_not_packet",
"score": 0.071,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"reusable_packaging"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [],
"exclusive_negative_concepts": [],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.429,
"negative_penalty": 0,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable",
"workflow",
"package"
]
},
"negative": {}
}
}
},
{
"prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.",
"family": "adversarial_debugging_collision",
"score": 0.0,
"predicted_trigger": false,
"expected_trigger": false,
"passed": true,
"score_detail": {
"mode": "semantic-intent",
"desired_positive_concepts": [
"incident_packet",
"incident_inputs",
"severity_assessment",
"comms_and_owners",
"governed_workflow",
"reusable_packaging"
],
"matched_desired_concepts": [
"reusable_packaging"
],
"extra_positive_concepts": [],
"matched_negative_concepts": [
"generic_debugging"
],
"exclusive_negative_concepts": [
"generic_debugging"
],
"semantic_coverage": 0.04,
"support_score": 0,
"lexical_support": 0.308,
"negative_penalty": 0.3,
"coverage_boost": 0.0,
"concept_evidence": {
"positive": {
"reusable_packaging": [
"reusable"
]
},
"negative": {
"generic_debugging": [
"fix the service"
]
}
}
}
}
]
}
},
"candidates": [
{
"id": "current",
"label": "Current",
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"strategy": "current",
"estimated_tokens": 37,
"dev": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
1,
-1.0,
-1.0,
-1.0,
-0.667,
37
],
"holdout": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "guardrail",
"label": "Guardrail",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
"strategy": "guardrail_template",
"estimated_tokens": 51,
"dev": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
1,
-1.0,
-1.0,
-1.0,
-0.667,
51
],
"holdout": {
"false_positives": 1,
"false_negatives": 1,
"precision": 0.5,
"recall": 0.5,
"near_neighbor_pass_rate": 0.5,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "balanced",
"label": "Balanced",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
"strategy": "balanced_template",
"estimated_tokens": 54,
"dev": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
1,
-1.0,
-1.0,
-1.0,
-0.667,
54
],
"holdout": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "boundary",
"label": "Boundary",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
"strategy": "boundary_template",
"estimated_tokens": 78,
"dev": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
1,
-1.0,
-1.0,
-1.0,
-0.667,
78
],
"holdout": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
},
{
"id": "artifact_aware",
"label": "Artifact Aware",
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.",
"strategy": "artifact_template",
"estimated_tokens": 78,
"dev": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.667,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"selection_key": [
0,
1,
-1.0,
-1.0,
-1.0,
-0.667,
78
],
"holdout": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
}
}
],
"selection_logic": {
"priority": [
"fewest false positives",
"fewest false negatives",
"highest near-neighbor pass rate",
"highest negative pass rate",
"highest precision",
"highest recall",
"shortest description"
]
},
"comparison": {
"winner_vs_current_dev": {
"baseline_false_positives": 0,
"baseline_false_negatives": 1,
"improved_false_positives": 0,
"improved_false_negatives": 1,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.667,
"improved_recall": 0.667
},
"winner_vs_current_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 1,
"improved_false_positives": 0,
"improved_false_negatives": 1,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.5,
"improved_recall": 0.5
},
"winner_vs_current_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_dev": {
"baseline_false_positives": 0,
"baseline_false_negatives": 1,
"improved_false_positives": 0,
"improved_false_negatives": 1,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.667,
"improved_recall": 0.667
},
"winner_vs_baseline_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 2,
"improved_false_positives": 0,
"improved_false_negatives": 1,
"false_positive_delta": 0,
"false_negative_delta": -1,
"baseline_precision": null,
"improved_precision": 1.0,
"baseline_recall": 0.0,
"improved_recall": 0.5
},
"winner_vs_baseline_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 1,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": -1,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 0.5,
"improved_recall": 1.0
},
"winner_vs_current_judge_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_judge_blind_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_current_adversarial_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
},
"winner_vs_baseline_adversarial_holdout": {
"baseline_false_positives": 0,
"baseline_false_negatives": 0,
"improved_false_positives": 0,
"improved_false_negatives": 0,
"false_positive_delta": 0,
"false_negative_delta": 0,
"baseline_precision": 1.0,
"improved_precision": 1.0,
"baseline_recall": 1.0,
"improved_recall": 1.0
}
},
"acceptance_gates": {
"selection_basis": "dev only",
"holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 2,
"precision": null,
"recall": 0.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"current_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.37,
"mean_non_trigger_score": 0.019,
"mean_near_neighbor_score": 0.037,
"min_positive_score": 0.08,
"max_non_trigger_score": 0.057,
"score_gap": 0.023,
"threshold_margin": -0.4,
"positive_threshold_buffer": -0.4,
"negative_threshold_buffer": 0.423,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "overlap"
},
"baseline_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.36,
"mean_non_trigger_score": 0.059,
"mean_near_neighbor_score": 0.119,
"min_positive_score": 0.321,
"max_non_trigger_score": 0.22,
"score_gap": 0.101,
"threshold_margin": -0.159,
"positive_threshold_buffer": -0.159,
"negative_threshold_buffer": 0.26,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
"risk_band": "overlap"
},
"winner_family_health": {
"family_count": 6,
"clean_family_count": 5,
"failing_families": [
{
"family": "packet_assembly",
"errors": 1,
"pass_rate": 0.0
}
],
"weakest_family": {
"family": "packet_assembly",
"pass_rate": 0.0,
"errors": 1
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 5,
"failing_families": [
{
"family": "packet_assembly",
"errors": 1,
"pass_rate": 0.0
}
],
"weakest_family": {
"family": "packet_assembly",
"pass_rate": 0.0,
"errors": 1
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 4,
"failing_families": [
{
"family": "governed_workflow",
"errors": 1,
"pass_rate": 0.0
},
{
"family": "packet_assembly",
"errors": 1,
"pass_rate": 0.0
}
],
"weakest_family": {
"family": "packet_assembly",
"pass_rate": 0.0,
"errors": 1
}
}
},
"blind_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 1,
"precision": 1.0,
"recall": 0.5,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
"risk_band": "healthy"
},
"current_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.902,
"mean_non_trigger_score": 0.168,
"mean_near_neighbor_score": 0.138,
"min_positive_score": 0.804,
"max_non_trigger_score": 0.386,
"score_gap": 0.418,
"threshold_margin": 0.094,
"positive_threshold_buffer": 0.324,
"negative_threshold_buffer": 0.094,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
"risk_band": "healthy"
},
"baseline_calibration": {
"threshold": 0.48,
"mean_positive_score": 0.625,
"mean_non_trigger_score": 0.151,
"mean_near_neighbor_score": 0.198,
"min_positive_score": 0.458,
"max_non_trigger_score": 0.395,
"score_gap": 0.063,
"threshold_margin": -0.022,
"positive_threshold_buffer": -0.022,
"negative_threshold_buffer": 0.085,
"boundary_case_count": 2,
"boundary_case_rate": 0.333,
"risk_band": "overlap"
},
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_summary_only",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_summary_only",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 5,
"failing_families": [
{
"family": "blind_governed_workflow",
"errors": 1,
"pass_rate": 0.0
}
],
"weakest_family": {
"family": "blind_governed_workflow",
"pass_rate": 0.0,
"errors": 1
}
}
},
"judge_blind_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.657,
"rubric_version": "blind-v1"
}
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.657,
"rubric_version": "blind-v1"
}
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0,
"judge_summary": {
"agreement_rate": 1.0,
"mean_confidence": 0.69,
"rubric_version": "blind-v1"
}
},
"winner_calibration": null,
"current_calibration": null,
"baseline_calibration": null,
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_summary_only",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_summary_only",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "blind_summary_only",
"pass_rate": 1.0,
"errors": 0
}
}
},
"adversarial_holdout_non_regression": {
"winner": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"current": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"baseline": {
"false_positives": 0,
"false_negatives": 0,
"precision": 1.0,
"recall": 1.0,
"near_neighbor_pass_rate": 1.0,
"should_not_trigger_pass_rate": 1.0
},
"winner_calibration": {
"threshold": 0.34,
"mean_positive_score": 0.827,
"mean_non_trigger_score": 0.027,
"mean_near_neighbor_score": 0.011,
"min_positive_score": 0.654,
"max_non_trigger_score": 0.056,
"score_gap": 0.598,
"threshold_margin": 0.284,
"positive_threshold_buffer": 0.314,
"negative_threshold_buffer": 0.284,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"current_calibration": {
"threshold": 0.34,
"mean_positive_score": 0.827,
"mean_non_trigger_score": 0.027,
"mean_near_neighbor_score": 0.011,
"min_positive_score": 0.654,
"max_non_trigger_score": 0.056,
"score_gap": 0.598,
"threshold_margin": 0.284,
"positive_threshold_buffer": 0.314,
"negative_threshold_buffer": 0.284,
"boundary_case_count": 0,
"boundary_case_rate": 0.0,
"risk_band": "healthy"
},
"baseline_calibration": {
"threshold": 0.34,
"mean_positive_score": 0.689,
"mean_non_trigger_score": 0.074,
"mean_near_neighbor_score": 0.035,
"min_positive_score": 0.387,
"max_non_trigger_score": 0.215,
"score_gap": 0.172,
"threshold_margin": 0.047,
"positive_threshold_buffer": 0.047,
"negative_threshold_buffer": 0.125,
"boundary_case_count": 1,
"boundary_case_rate": 0.167,
"risk_band": "watch"
},
"winner_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_single_update_collision",
"pass_rate": 1.0,
"errors": 0
}
},
"current_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_single_update_collision",
"pass_rate": 1.0,
"errors": 0
}
},
"baseline_family_health": {
"family_count": 6,
"clean_family_count": 6,
"failing_families": [],
"weakest_family": {
"family": "adversarial_single_update_collision",
"pass_rate": 1.0,
"errors": 0
}
}
}
},
"summary": {
"winner_label": "Current",
"winner_tokens": 37,
"current_tokens": 37,
"winner_dev_total_errors": 1,
"current_dev_total_errors": 1,
"winner_holdout_total_errors": 1,
"current_holdout_total_errors": 1,
"winner_blind_holdout_total_errors": 0,
"current_blind_holdout_total_errors": 0,
"winner_judge_blind_holdout_total_errors": 0,
"current_judge_blind_holdout_total_errors": 0,
"winner_adversarial_holdout_total_errors": 0,
"current_adversarial_holdout_total_errors": 0,
"winner_judge_blind_agreement_rate": 1.0,
"winner_adversarial_risk_band": "healthy",
"winner_adversarial_score_gap": 0.598,
"candidate_count": 5,
"baseline_tokens": 93,
"baseline_dev_total_errors": 1,
"baseline_holdout_total_errors": 2,
"baseline_blind_holdout_total_errors": 1,
"baseline_judge_blind_holdout_total_errors": 0,
"baseline_adversarial_holdout_total_errors": 0
}
}
Reference in New Issue View Git Blame Copy Permalink