yaojingang
2553 lines
79 KiB
JSON
2553 lines
79 KiB
JSON
{
|
|
"current_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"current_candidate": {
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 37,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
37
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
"baseline": {
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.",
|
|
"estimated_tokens": 93,
|
|
"dev": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 3,
|
|
"passed": 2,
|
|
"pass_rate": 0.667
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"governed_workflow": {
|
|
"total": 2,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.5
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"generic_debugging": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "governed_workflow",
|
|
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
|
|
"score": 0.392,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"family": "packet_assembly",
|
|
"score": 0.686,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_inputs",
|
|
"incident_packet"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.62,
|
|
"support_score": 0,
|
|
"lexical_support": 0.692,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"incident notes",
|
|
"alerts",
|
|
"transcripts"
|
|
],
|
|
"incident_packet": [
|
|
"incident command packet"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
|
|
"family": "governed_workflow",
|
|
"score": 0.791,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs",
|
|
"incident_packet",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.74,
|
|
"support_score": 0,
|
|
"lexical_support": 0.625,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"outage workflow"
|
|
],
|
|
"incident_packet": [
|
|
"incident review skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"skill"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
|
|
"family": "governed_workflow",
|
|
"score": 0.392,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.32,
|
|
"support_score": 0,
|
|
"lexical_support": 0.467,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain what incident severity means.",
|
|
"family": "explain_only",
|
|
"score": 0.198,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.18,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Draft one short outage update for customers.",
|
|
"family": "one_off_update",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.286,
|
|
"negative_penalty": 0.34,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"one_off_update": [
|
|
"draft one short outage update"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Debug the service outage and find the root cause.",
|
|
"family": "generic_debugging",
|
|
"score": 0.01,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize this incident but do not package it as a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.063,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.04,
|
|
"support_score": 0,
|
|
"lexical_support": 0.333,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.14,
|
|
"support_score": 0,
|
|
"lexical_support": 0.286,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 2,
|
|
"precision": null,
|
|
"recall": 0.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 0,
|
|
"pass_rate": 0.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"governed_workflow": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "governed_workflow",
|
|
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
|
|
"score": 0.394,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"matched_negative_concepts": []
|
|
},
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"score": 0.321,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
|
|
"family": "governed_workflow",
|
|
"score": 0.394,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.32,
|
|
"support_score": 0,
|
|
"lexical_support": 0.5,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"family": "packet_assembly",
|
|
"score": 0.321,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.24,
|
|
"support_score": 0,
|
|
"lexical_support": 0.75,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"alerts"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write a single executive summary of this outage only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm what the team could do better next time.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Draft a one-off internal status update from these incident notes.",
|
|
"family": "one_off_update",
|
|
"score": 0.216,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.2,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain how incident command works at a high level.",
|
|
"family": "explain_only",
|
|
"score": 0.018,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"winner": {
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 37,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
37
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
"winner_dev_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 3,
|
|
"passed": 2,
|
|
"pass_rate": 0.667
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"governed_workflow": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"generic_debugging": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"score": 0.348,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"family": "packet_assembly",
|
|
"score": 0.348,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"incident_packet"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.227,
|
|
"support_score": 0.52,
|
|
"lexical_support": 0.231,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"incident notes",
|
|
"alerts",
|
|
"transcripts"
|
|
],
|
|
"incident_packet": [
|
|
"incident command packet"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
|
|
"family": "governed_workflow",
|
|
"score": 0.521,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"incident_packet",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.409,
|
|
"support_score": 0.56,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"outage workflow"
|
|
],
|
|
"incident_packet": [
|
|
"incident review skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"skill"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
|
|
"family": "governed_workflow",
|
|
"score": 0.651,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.636,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain what incident severity means.",
|
|
"family": "explain_only",
|
|
"score": 0.408,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.409,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Draft one short outage update for customers.",
|
|
"family": "one_off_update",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.34,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"one_off_update": [
|
|
"draft one short outage update"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Debug the service outage and find the root cause.",
|
|
"family": "generic_debugging",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize this incident but do not package it as a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.017,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.227,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.214,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"winner_holdout_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 1,
|
|
"pass_rate": 0.5
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"score": 0.08,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
|
|
"family": "governed_workflow",
|
|
"score": 0.655,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.636,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"family": "packet_assembly",
|
|
"score": 0.08,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"alerts"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write a single executive summary of this outage only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm what the team could do better next time.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Draft a one-off internal status update from these incident notes.",
|
|
"family": "one_off_update",
|
|
"score": 0.058,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain how incident command works at a high level.",
|
|
"family": "explain_only",
|
|
"score": 0.018,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_dev_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 3,
|
|
"passed": 2,
|
|
"pass_rate": 0.667
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"governed_workflow": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"generic_debugging": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"score": 0.348,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"family": "packet_assembly",
|
|
"score": 0.348,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"incident_packet"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.227,
|
|
"support_score": 0.52,
|
|
"lexical_support": 0.231,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"incident notes",
|
|
"alerts",
|
|
"transcripts"
|
|
],
|
|
"incident_packet": [
|
|
"incident command packet"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
|
|
"family": "governed_workflow",
|
|
"score": 0.521,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"incident_packet",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.409,
|
|
"support_score": 0.56,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"outage workflow"
|
|
],
|
|
"incident_packet": [
|
|
"incident review skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"skill"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
|
|
"family": "governed_workflow",
|
|
"score": 0.651,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.636,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain what incident severity means.",
|
|
"family": "explain_only",
|
|
"score": 0.408,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.409,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Draft one short outage update for customers.",
|
|
"family": "one_off_update",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.34,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"one_off_update": [
|
|
"draft one short outage update"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Debug the service outage and find the root cause.",
|
|
"family": "generic_debugging",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize this incident but do not package it as a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.017,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.227,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.214,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_holdout_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 1,
|
|
"pass_rate": 0.5
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"score": 0.08,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
|
|
"family": "governed_workflow",
|
|
"score": 0.655,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.636,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"family": "packet_assembly",
|
|
"score": 0.08,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"alerts"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write a single executive summary of this outage only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm what the team could do better next time.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Draft a one-off internal status update from these incident notes.",
|
|
"family": "one_off_update",
|
|
"score": 0.058,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain how incident command works at a high level.",
|
|
"family": "explain_only",
|
|
"score": 0.018,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"candidates": [
|
|
{
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 37,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
37
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "guardrail",
|
|
"label": "Guardrail",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
|
|
"strategy": "guardrail_template",
|
|
"estimated_tokens": 51,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
51
|
|
],
|
|
"holdout": {
|
|
"false_positives": 1,
|
|
"false_negatives": 1,
|
|
"precision": 0.5,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 0.5,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "balanced",
|
|
"label": "Balanced",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "balanced_template",
|
|
"estimated_tokens": 54,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
54
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 2,
|
|
"precision": null,
|
|
"recall": 0.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "boundary",
|
|
"label": "Boundary",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
|
|
"strategy": "boundary_template",
|
|
"estimated_tokens": 78,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
78
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "artifact_aware",
|
|
"label": "Artifact Aware",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "artifact_template",
|
|
"estimated_tokens": 78,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
78
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
}
|
|
],
|
|
"selection_logic": {
|
|
"priority": [
|
|
"fewest false positives",
|
|
"fewest false negatives",
|
|
"highest near-neighbor pass rate",
|
|
"highest negative pass rate",
|
|
"highest precision",
|
|
"highest recall",
|
|
"shortest description"
|
|
]
|
|
},
|
|
"comparison": {
|
|
"winner_vs_current_dev": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 1,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 1,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 0.667,
|
|
"improved_recall": 0.667
|
|
},
|
|
"winner_vs_current_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 1,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 1,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 0.5,
|
|
"improved_recall": 0.5
|
|
},
|
|
"winner_vs_baseline_dev": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 1,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 1,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 0.667,
|
|
"improved_recall": 0.667
|
|
},
|
|
"winner_vs_baseline_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 2,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 1,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": -1,
|
|
"baseline_precision": null,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 0.0,
|
|
"improved_recall": 0.5
|
|
}
|
|
},
|
|
"summary": {
|
|
"winner_label": "Current",
|
|
"winner_tokens": 37,
|
|
"current_tokens": 37,
|
|
"winner_dev_total_errors": 1,
|
|
"current_dev_total_errors": 1,
|
|
"winner_holdout_total_errors": 1,
|
|
"current_holdout_total_errors": 1,
|
|
"candidate_count": 5,
|
|
"baseline_tokens": 93,
|
|
"baseline_dev_total_errors": 1,
|
|
"baseline_holdout_total_errors": 2
|
|
}
|
|
}
|