YAO
31ce04c655
Merge the beta-ready Yao Meta Skill architecture, report, evidence gate, and release-boundary updates.\n\nRelease boundary: beta/public testing is allowed; formal world-class, fully reviewed, or superiority claims remain blocked until the pending evidence gates are accepted.
6262 lines
192 KiB
JSON
6262 lines
192 KiB
JSON
{
|
|
"current_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"current_candidate": {
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 37,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
37
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
"baseline": {
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.",
|
|
"estimated_tokens": 93,
|
|
"dev": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"governed_workflow": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"generic_debugging": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"family": "packet_assembly",
|
|
"score": 0.686,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_inputs",
|
|
"incident_packet"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.62,
|
|
"support_score": 0,
|
|
"lexical_support": 0.692,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"incident notes",
|
|
"alerts",
|
|
"transcripts"
|
|
],
|
|
"incident_packet": [
|
|
"incident command packet"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
|
|
"family": "governed_workflow",
|
|
"score": 0.791,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs",
|
|
"incident_packet",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.74,
|
|
"support_score": 0,
|
|
"lexical_support": 0.625,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"outage workflow"
|
|
],
|
|
"incident_packet": [
|
|
"incident review skill",
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"skill"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
|
|
"family": "governed_workflow",
|
|
"score": 0.539,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.48,
|
|
"support_score": 0,
|
|
"lexical_support": 0.467,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owners"
|
|
],
|
|
"governed_workflow": [
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain what incident severity means.",
|
|
"family": "explain_only",
|
|
"score": 0.198,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.18,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Draft one short outage update for customers.",
|
|
"family": "one_off_update",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.286,
|
|
"negative_penalty": 0.34,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"one_off_update": [
|
|
"draft one short outage update"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Debug the service outage and find the root cause.",
|
|
"family": "generic_debugging",
|
|
"score": 0.01,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize this incident but do not package it as a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.04,
|
|
"support_score": 0,
|
|
"lexical_support": 0.417,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.14,
|
|
"support_score": 0,
|
|
"lexical_support": 0.429,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 1,
|
|
"pass_rate": 0.5
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"score": 0.321,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
|
|
"family": "governed_workflow",
|
|
"score": 0.694,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.64,
|
|
"support_score": 0,
|
|
"lexical_support": 0.562,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"standardize"
|
|
],
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"family": "packet_assembly",
|
|
"score": 0.321,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.24,
|
|
"support_score": 0,
|
|
"lexical_support": 0.75,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"alerts"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write a single executive summary of this outage only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm what the team could do better next time.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.111,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Draft a one-off internal status update from these incident notes.",
|
|
"family": "one_off_update",
|
|
"score": 0.22,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.2,
|
|
"support_score": 0,
|
|
"lexical_support": 0.455,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain how incident command works at a high level.",
|
|
"family": "explain_only",
|
|
"score": 0.018,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"winner": {
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 37,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
37
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
"winner_dev_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"governed_workflow": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"generic_debugging": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"family": "packet_assembly",
|
|
"score": 0.617,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.553,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.231,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"incident notes",
|
|
"alerts",
|
|
"transcripts"
|
|
],
|
|
"incident_packet": [
|
|
"incident command packet"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
|
|
"family": "governed_workflow",
|
|
"score": 0.73,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.658,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"outage workflow"
|
|
],
|
|
"incident_packet": [
|
|
"incident review skill",
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"skill"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
|
|
"family": "governed_workflow",
|
|
"score": 0.619,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.579,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owners"
|
|
],
|
|
"governed_workflow": [
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain what incident severity means.",
|
|
"family": "explain_only",
|
|
"score": 0.25,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.237,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Draft one short outage update for customers.",
|
|
"family": "one_off_update",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.34,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"one_off_update": [
|
|
"draft one short outage update"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Debug the service outage and find the root cause.",
|
|
"family": "generic_debugging",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize this incident but do not package it as a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.132,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.286,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"winner_holdout_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 1,
|
|
"pass_rate": 0.5
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"score": 0.08,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
|
|
"family": "governed_workflow",
|
|
"score": 0.821,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.789,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"standardize"
|
|
],
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"family": "packet_assembly",
|
|
"score": 0.08,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"alerts"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write a single executive summary of this outage only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm what the team could do better next time.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Draft a one-off internal status update from these incident notes.",
|
|
"family": "one_off_update",
|
|
"score": 0.057,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain how incident command works at a high level.",
|
|
"family": "explain_only",
|
|
"score": 0.018,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_dev_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 3,
|
|
"passed": 3,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"governed_workflow": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"generic_debugging": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alerts, incident notes, and responder transcripts into a governed incident command packet.",
|
|
"family": "packet_assembly",
|
|
"score": 0.617,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.553,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.231,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"incident notes",
|
|
"alerts",
|
|
"transcripts"
|
|
],
|
|
"incident_packet": [
|
|
"incident command packet"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Create a reusable incident review skill from this outage workflow with severity assessment and stakeholder communication.",
|
|
"family": "governed_workflow",
|
|
"score": 0.73,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.658,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"outage workflow"
|
|
],
|
|
"incident_packet": [
|
|
"incident review skill",
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"skill"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Package this incident process into a maintained operational skill with severity, owners, and communication blocks.",
|
|
"family": "governed_workflow",
|
|
"score": 0.619,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.579,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owners"
|
|
],
|
|
"governed_workflow": [
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain what incident severity means.",
|
|
"family": "explain_only",
|
|
"score": 0.25,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.237,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Draft one short outage update for customers.",
|
|
"family": "one_off_update",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"one_off_update"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.34,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"one_off_update": [
|
|
"draft one short outage update"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Debug the service outage and find the root cause.",
|
|
"family": "generic_debugging",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize this incident but do not package it as a reusable skill.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Help brainstorm improvements to incident handling, but do not build the governed workflow yet.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.132,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.286,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_holdout_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 1,
|
|
"pass_rate": 0.5
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"packet_assembly": {
|
|
"total": 1,
|
|
"passed": 0,
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"pass_rate": 0.0
|
|
},
|
|
"summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [
|
|
{
|
|
"bucket": "should_trigger",
|
|
"family": "packet_assembly",
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"score": 0.08,
|
|
"reason": "false_negative",
|
|
"matched_desired_concepts": [],
|
|
"matched_negative_concepts": []
|
|
}
|
|
],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Standardize this incident review process into a governed packet workflow with severity rationale and stakeholder updates.",
|
|
"family": "governed_workflow",
|
|
"score": 0.821,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.789,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.312,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"standardize"
|
|
],
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Build a reusable incident command skill from alerts, timelines, and responder notes.",
|
|
"family": "packet_assembly",
|
|
"score": 0.08,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": true,
|
|
"passed": false,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.25,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"alerts"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"skill"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write a single executive summary of this outage only.",
|
|
"family": "summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm what the team could do better next time.",
|
|
"family": "brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.0,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Draft a one-off internal status update from these incident notes.",
|
|
"family": "one_off_update",
|
|
"score": 0.057,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain how incident command works at a high level.",
|
|
"family": "explain_only",
|
|
"score": 0.018,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.222,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"winner_blind_holdout_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
|
|
"family": "blind_packet_assembly",
|
|
"score": 0.994,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 1.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.176,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
|
|
"family": "blind_governed_workflow",
|
|
"score": 0.645,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.579,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.533,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"incident communication"
|
|
],
|
|
"governed_workflow": [
|
|
"standardize"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"outage review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain how severity levels are decided during incidents.",
|
|
"family": "blind_explain_only",
|
|
"score": 0.228,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.237,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Write a single incident update for customers from these notes.",
|
|
"family": "blind_one_off_update",
|
|
"score": 0.008,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
|
|
"family": "blind_summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.132,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
|
|
"family": "blind_brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_blind_holdout_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
|
|
"family": "blind_packet_assembly",
|
|
"score": 0.994,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 1.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.176,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
|
|
"family": "blind_governed_workflow",
|
|
"score": 0.645,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.579,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.533,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"incident communication"
|
|
],
|
|
"governed_workflow": [
|
|
"standardize"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"outage review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain how severity levels are decided during incidents.",
|
|
"family": "blind_explain_only",
|
|
"score": 0.228,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.237,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Write a single incident update for customers from these notes.",
|
|
"family": "blind_one_off_update",
|
|
"score": 0.008,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.1,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
|
|
"family": "blind_summary_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.132,
|
|
"support_score": 0.24,
|
|
"lexical_support": 0.091,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
|
|
"family": "blind_brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.083,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"baseline_blind_holdout_report": {
|
|
"threshold": 0.48,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
|
|
"family": "blind_packet_assembly",
|
|
"score": 0.792,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.76,
|
|
"support_score": 0,
|
|
"lexical_support": 0.412,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
|
|
"family": "blind_governed_workflow",
|
|
"score": 0.55,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.48,
|
|
"support_score": 0,
|
|
"lexical_support": 0.6,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"incident communication"
|
|
],
|
|
"governed_workflow": [
|
|
"standardize"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"outage review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain how severity levels are decided during incidents.",
|
|
"family": "blind_explain_only",
|
|
"score": 0.176,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.18,
|
|
"support_score": 0,
|
|
"lexical_support": 0.125,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Write a single incident update for customers from these notes.",
|
|
"family": "blind_one_off_update",
|
|
"score": 0.032,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
|
|
"family": "blind_summary_only",
|
|
"score": 0.005,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_inputs",
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.34,
|
|
"support_score": 0,
|
|
"lexical_support": 0.273,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
|
|
"family": "blind_brainstorm_only",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0,
|
|
"lexical_support": 0.167,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"winner_judge_blind_holdout_report": {
|
|
"judge": "rubric-blind-v1",
|
|
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
|
|
"family": "blind_packet_assembly",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 4,
|
|
"negative_vote": 0,
|
|
"margin": 4,
|
|
"confidence": 0.82,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
|
|
"family": "blind_governed_workflow",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners"
|
|
],
|
|
"trigger_action_hits": [
|
|
"run severity assessment",
|
|
"assemble incident communication"
|
|
],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 6,
|
|
"negative_vote": 0,
|
|
"margin": 6,
|
|
"confidence": 1.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"incident communication"
|
|
],
|
|
"governed_workflow": [
|
|
"standardize"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"outage review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain how severity levels are decided during incidents.",
|
|
"family": "blind_explain_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 0,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"explain"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 1,
|
|
"margin": 0,
|
|
"confidence": 0.5,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Write a single incident update for customers from these notes.",
|
|
"family": "blind_one_off_update",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 1,
|
|
"negative_vote": 0,
|
|
"margin": 1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
|
|
"family": "blind_summary_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"summarize"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 3,
|
|
"margin": -2,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
|
|
"family": "blind_brainstorm_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"brainstorm"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 3,
|
|
"margin": -2,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"current_judge_blind_holdout_report": {
|
|
"judge": "rubric-blind-v1",
|
|
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
|
|
"family": "blind_packet_assembly",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 4,
|
|
"negative_vote": 0,
|
|
"margin": 4,
|
|
"confidence": 0.82,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
|
|
"family": "blind_governed_workflow",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners"
|
|
],
|
|
"trigger_action_hits": [
|
|
"run severity assessment",
|
|
"assemble incident communication"
|
|
],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 6,
|
|
"negative_vote": 0,
|
|
"margin": 6,
|
|
"confidence": 1.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"incident communication"
|
|
],
|
|
"governed_workflow": [
|
|
"standardize"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"outage review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain how severity levels are decided during incidents.",
|
|
"family": "blind_explain_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 0,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"explain"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 1,
|
|
"margin": 0,
|
|
"confidence": 0.5,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Write a single incident update for customers from these notes.",
|
|
"family": "blind_one_off_update",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 1,
|
|
"negative_vote": 0,
|
|
"margin": 1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
|
|
"family": "blind_summary_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"governed_workflow"
|
|
],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"summarize"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 3,
|
|
"margin": -2,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
|
|
"family": "blind_brainstorm_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"brainstorm"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 3,
|
|
"margin": -2,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"baseline_judge_blind_holdout_report": {
|
|
"judge": "rubric-blind-v1",
|
|
"judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"blind_packet_assembly": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_governed_workflow": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_explain_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_one_off_update": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_summary_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"blind_brainstorm_only": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Turn these alert logs and responder notes into a governed incident packet with severity rationale and owner actions.",
|
|
"family": "blind_packet_assembly",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 2,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 4,
|
|
"negative_vote": 0,
|
|
"margin": 4,
|
|
"confidence": 0.82,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Standardize this outage review, run severity assessment, and assemble incident communication into a reusable incident workflow.",
|
|
"family": "blind_governed_workflow",
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"severity_assessment",
|
|
"comms_and_owners"
|
|
],
|
|
"trigger_action_hits": [
|
|
"run severity assessment",
|
|
"assemble incident communication"
|
|
],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 6,
|
|
"negative_vote": 0,
|
|
"margin": 6,
|
|
"confidence": 1.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"incident communication"
|
|
],
|
|
"governed_workflow": [
|
|
"standardize"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity",
|
|
"outage review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Explain how severity levels are decided during incidents.",
|
|
"family": "blind_explain_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 0,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"explain"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 1,
|
|
"margin": 0,
|
|
"confidence": 0.5,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Write a single incident update for customers from these notes.",
|
|
"family": "blind_one_off_update",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [],
|
|
"positive_vote": 1,
|
|
"negative_vote": 0,
|
|
"margin": 1,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Summarize the outage timeline, but do not package a governed workflow.",
|
|
"family": "blind_summary_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [
|
|
"incident_inputs",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"strong_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"summarize"
|
|
],
|
|
"positive_vote": 3,
|
|
"negative_vote": 3,
|
|
"margin": 0,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"governed",
|
|
"governed workflow"
|
|
],
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"reusable_packaging": [
|
|
"workflow",
|
|
"package"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"do not package"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Brainstorm how incident handling could improve next quarter without building the packet.",
|
|
"family": "blind_brainstorm_only",
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"judge_detail": {
|
|
"mode": "judge-rubric",
|
|
"focused_positive_concepts": [],
|
|
"strong_positive_concepts": [],
|
|
"trigger_action_hits": [],
|
|
"input_hits": [],
|
|
"artifact_hits": [],
|
|
"capability_overlap": 1,
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"nonexclusive_negative_concepts": [],
|
|
"exclusion_hits": [],
|
|
"generic_exclusion_hits": [
|
|
"brainstorm"
|
|
],
|
|
"positive_vote": 1,
|
|
"negative_vote": 3,
|
|
"margin": -2,
|
|
"confidence": 0.58,
|
|
"concept_evidence": {
|
|
"positive": {},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"brainstorm"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"winner_adversarial_holdout_report": {
|
|
"threshold": 0.34,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"adversarial_governed_packet_from_noise": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_operational_skill_route": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_single_update_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_explain_and_summary_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_future_playbook_not_packet": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_debugging_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.",
|
|
"family": "adversarial_governed_packet_from_noise",
|
|
"score": 1.0,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 1.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.263,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"alerts",
|
|
"transcripts",
|
|
"stakeholder requirements"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.",
|
|
"family": "adversarial_operational_skill_route",
|
|
"score": 0.815,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.789,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.235,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"standardize",
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write one short customer outage update from these incident notes and stop there.",
|
|
"family": "adversarial_single_update_collision",
|
|
"score": 0.056,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.077,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain incident severity and summarize the outage timeline for training.",
|
|
"family": "adversarial_explain_and_summary_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.237,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain incident severity"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.",
|
|
"family": "adversarial_future_playbook_not_packet",
|
|
"score": 0.277,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.658,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"sketch a future",
|
|
"for later",
|
|
"do not package",
|
|
"do not package a reusable workflow"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.",
|
|
"family": "adversarial_debugging_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"generic_debugging"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"generic_debugging"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.154,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"generic_debugging": [
|
|
"fix the service"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"current_adversarial_holdout_report": {
|
|
"threshold": 0.34,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"adversarial_governed_packet_from_noise": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_operational_skill_route": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_single_update_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_explain_and_summary_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_future_playbook_not_packet": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_debugging_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.",
|
|
"family": "adversarial_governed_packet_from_noise",
|
|
"score": 1.0,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 1.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.263,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"alerts",
|
|
"transcripts",
|
|
"stakeholder requirements"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.",
|
|
"family": "adversarial_operational_skill_route",
|
|
"score": 0.815,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.789,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.235,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"standardize",
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write one short customer outage update from these incident notes and stop there.",
|
|
"family": "adversarial_single_update_collision",
|
|
"score": 0.056,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.077,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain incident severity and summarize the outage timeline for training.",
|
|
"family": "adversarial_explain_and_summary_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.237,
|
|
"support_score": 0.2,
|
|
"lexical_support": 0.2,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain incident severity"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.",
|
|
"family": "adversarial_future_playbook_not_packet",
|
|
"score": 0.277,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.658,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.143,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"sketch a future",
|
|
"for later",
|
|
"do not package",
|
|
"do not package a reusable workflow"
|
|
]
|
|
}
|
|
}
|
|
},
|
|
"boundary_case": true
|
|
},
|
|
{
|
|
"prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.",
|
|
"family": "adversarial_debugging_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow"
|
|
],
|
|
"matched_desired_concepts": [],
|
|
"extra_positive_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"matched_negative_concepts": [
|
|
"generic_debugging"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"generic_debugging"
|
|
],
|
|
"semantic_coverage": 0.0,
|
|
"support_score": 0.04,
|
|
"lexical_support": 0.154,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"generic_debugging": [
|
|
"fix the service"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"baseline_adversarial_holdout_report": {
|
|
"threshold": 0.34,
|
|
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.",
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"bucket_stats": {
|
|
"should_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"should_not_trigger": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
},
|
|
"near_neighbor": {
|
|
"total": 2,
|
|
"passed": 2,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"family_stats": {
|
|
"adversarial_governed_packet_from_noise": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_operational_skill_route": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_single_update_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_explain_and_summary_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_future_playbook_not_packet": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
},
|
|
"adversarial_debugging_collision": {
|
|
"total": 1,
|
|
"passed": 1,
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"pass_rate": 1.0
|
|
}
|
|
},
|
|
"misfires": [],
|
|
"results": {
|
|
"should_trigger": [
|
|
{
|
|
"prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.",
|
|
"family": "adversarial_governed_packet_from_noise",
|
|
"score": 0.99,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"incident_inputs",
|
|
"incident_packet",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.96,
|
|
"support_score": 0,
|
|
"lexical_support": 0.579,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"comms_and_owners": [
|
|
"owner",
|
|
"owner actions"
|
|
],
|
|
"governed_workflow": [
|
|
"governed"
|
|
],
|
|
"incident_inputs": [
|
|
"alerts",
|
|
"transcripts",
|
|
"stakeholder requirements"
|
|
],
|
|
"incident_packet": [
|
|
"governed incident packet"
|
|
],
|
|
"severity_assessment": [
|
|
"severity assessment",
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.",
|
|
"family": "adversarial_operational_skill_route",
|
|
"score": 0.682,
|
|
"predicted_trigger": true,
|
|
"expected_trigger": true,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"governed_workflow",
|
|
"incident_packet",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.64,
|
|
"support_score": 0,
|
|
"lexical_support": 0.412,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"governed_workflow": [
|
|
"standardize",
|
|
"maintained",
|
|
"operational skill"
|
|
],
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"skill",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"severity",
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"should_not_trigger": [
|
|
{
|
|
"prompt": "Write one short customer outage update from these incident notes and stop there.",
|
|
"family": "adversarial_single_update_collision",
|
|
"score": 0.215,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [],
|
|
"exclusive_negative_concepts": [],
|
|
"semantic_coverage": 0.2,
|
|
"support_score": 0,
|
|
"lexical_support": 0.385,
|
|
"negative_penalty": 0,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"incident notes"
|
|
]
|
|
},
|
|
"negative": {}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Explain incident severity and summarize the outage timeline for training.",
|
|
"family": "adversarial_explain_and_summary_collision",
|
|
"score": 0.012,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_inputs",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"explain_only"
|
|
],
|
|
"semantic_coverage": 0.38,
|
|
"support_score": 0,
|
|
"lexical_support": 0.4,
|
|
"negative_penalty": 0.26,
|
|
"coverage_boost": 0.04,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_inputs": [
|
|
"timeline"
|
|
],
|
|
"severity_assessment": [
|
|
"severity"
|
|
]
|
|
},
|
|
"negative": {
|
|
"explain_only": [
|
|
"explain incident severity"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"near_neighbor": [
|
|
{
|
|
"prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.",
|
|
"family": "adversarial_future_playbook_not_packet",
|
|
"score": 0.201,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"incident_packet",
|
|
"reusable_packaging",
|
|
"severity_assessment"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"brainstorm_only"
|
|
],
|
|
"semantic_coverage": 0.54,
|
|
"support_score": 0,
|
|
"lexical_support": 0.429,
|
|
"negative_penalty": 0.24,
|
|
"coverage_boost": 0.06,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"incident_packet": [
|
|
"incident review"
|
|
],
|
|
"reusable_packaging": [
|
|
"reusable",
|
|
"workflow",
|
|
"package"
|
|
],
|
|
"severity_assessment": [
|
|
"incident review"
|
|
]
|
|
},
|
|
"negative": {
|
|
"brainstorm_only": [
|
|
"sketch a future",
|
|
"for later",
|
|
"do not package",
|
|
"do not package a reusable workflow"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.",
|
|
"family": "adversarial_debugging_collision",
|
|
"score": 0.0,
|
|
"predicted_trigger": false,
|
|
"expected_trigger": false,
|
|
"passed": true,
|
|
"score_detail": {
|
|
"mode": "semantic-intent",
|
|
"desired_positive_concepts": [
|
|
"incident_packet",
|
|
"incident_inputs",
|
|
"severity_assessment",
|
|
"comms_and_owners",
|
|
"governed_workflow",
|
|
"reusable_packaging"
|
|
],
|
|
"matched_desired_concepts": [
|
|
"reusable_packaging"
|
|
],
|
|
"extra_positive_concepts": [],
|
|
"matched_negative_concepts": [
|
|
"generic_debugging"
|
|
],
|
|
"exclusive_negative_concepts": [
|
|
"generic_debugging"
|
|
],
|
|
"semantic_coverage": 0.04,
|
|
"support_score": 0,
|
|
"lexical_support": 0.308,
|
|
"negative_penalty": 0.3,
|
|
"coverage_boost": 0.0,
|
|
"concept_evidence": {
|
|
"positive": {
|
|
"reusable_packaging": [
|
|
"reusable"
|
|
]
|
|
},
|
|
"negative": {
|
|
"generic_debugging": [
|
|
"fix the service"
|
|
]
|
|
}
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"candidates": [
|
|
{
|
|
"id": "current",
|
|
"label": "Current",
|
|
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "current",
|
|
"estimated_tokens": 37,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
37
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "balanced",
|
|
"label": "Balanced",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "balanced_template",
|
|
"estimated_tokens": 54,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
54
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "boundary",
|
|
"label": "Boundary",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
|
|
"strategy": "boundary_template",
|
|
"estimated_tokens": 78,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
78
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "artifact_aware",
|
|
"label": "Artifact Aware",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"strategy": "artifact_template",
|
|
"estimated_tokens": 78,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
78
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
},
|
|
{
|
|
"id": "guardrail",
|
|
"label": "Guardrail",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
|
|
"strategy": "guardrail_template",
|
|
"estimated_tokens": 51,
|
|
"dev": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.667,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"selection_key": [
|
|
0,
|
|
1,
|
|
-1.0,
|
|
-1.0,
|
|
-1.0,
|
|
-0.667,
|
|
51
|
|
],
|
|
"holdout": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
}
|
|
}
|
|
],
|
|
"selection_logic": {
|
|
"priority": [
|
|
"fewest false positives",
|
|
"fewest false negatives",
|
|
"highest near-neighbor pass rate",
|
|
"highest negative pass rate",
|
|
"highest precision",
|
|
"highest recall",
|
|
"shortest description"
|
|
]
|
|
},
|
|
"comparison": {
|
|
"winner_vs_current_dev": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_current_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 1,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 1,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 0.5,
|
|
"improved_recall": 0.5
|
|
},
|
|
"winner_vs_current_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_dev": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 1,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 1,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 0.5,
|
|
"improved_recall": 0.5
|
|
},
|
|
"winner_vs_baseline_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_current_judge_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_judge_blind_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_current_adversarial_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
},
|
|
"winner_vs_baseline_adversarial_holdout": {
|
|
"baseline_false_positives": 0,
|
|
"baseline_false_negatives": 0,
|
|
"improved_false_positives": 0,
|
|
"improved_false_negatives": 0,
|
|
"false_positive_delta": 0,
|
|
"false_negative_delta": 0,
|
|
"baseline_precision": 1.0,
|
|
"improved_precision": 1.0,
|
|
"baseline_recall": 1.0,
|
|
"improved_recall": 1.0
|
|
}
|
|
},
|
|
"acceptance_gates": {
|
|
"selection_basis": "dev only",
|
|
"holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 1,
|
|
"precision": 1.0,
|
|
"recall": 0.5,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"winner_calibration": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.45,
|
|
"mean_non_trigger_score": 0.019,
|
|
"mean_near_neighbor_score": 0.037,
|
|
"min_positive_score": 0.08,
|
|
"max_non_trigger_score": 0.057,
|
|
"score_gap": 0.023,
|
|
"threshold_margin": -0.4,
|
|
"positive_threshold_buffer": -0.4,
|
|
"negative_threshold_buffer": 0.423,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "overlap"
|
|
},
|
|
"current_calibration": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.45,
|
|
"mean_non_trigger_score": 0.019,
|
|
"mean_near_neighbor_score": 0.037,
|
|
"min_positive_score": 0.08,
|
|
"max_non_trigger_score": 0.057,
|
|
"score_gap": 0.023,
|
|
"threshold_margin": -0.4,
|
|
"positive_threshold_buffer": -0.4,
|
|
"negative_threshold_buffer": 0.423,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "overlap"
|
|
},
|
|
"baseline_calibration": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.507,
|
|
"mean_non_trigger_score": 0.059,
|
|
"mean_near_neighbor_score": 0.119,
|
|
"min_positive_score": 0.321,
|
|
"max_non_trigger_score": 0.22,
|
|
"score_gap": 0.101,
|
|
"threshold_margin": -0.159,
|
|
"positive_threshold_buffer": -0.159,
|
|
"negative_threshold_buffer": 0.26,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "overlap"
|
|
},
|
|
"winner_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 5,
|
|
"failing_families": [
|
|
{
|
|
"family": "packet_assembly",
|
|
"errors": 1,
|
|
"pass_rate": 0.0
|
|
}
|
|
],
|
|
"weakest_family": {
|
|
"family": "packet_assembly",
|
|
"pass_rate": 0.0,
|
|
"errors": 1
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 5,
|
|
"failing_families": [
|
|
{
|
|
"family": "packet_assembly",
|
|
"errors": 1,
|
|
"pass_rate": 0.0
|
|
}
|
|
],
|
|
"weakest_family": {
|
|
"family": "packet_assembly",
|
|
"pass_rate": 0.0,
|
|
"errors": 1
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 5,
|
|
"failing_families": [
|
|
{
|
|
"family": "packet_assembly",
|
|
"errors": 1,
|
|
"pass_rate": 0.0
|
|
}
|
|
],
|
|
"weakest_family": {
|
|
"family": "packet_assembly",
|
|
"pass_rate": 0.0,
|
|
"errors": 1
|
|
}
|
|
}
|
|
},
|
|
"blind_holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"winner_calibration": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.82,
|
|
"mean_non_trigger_score": 0.059,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.645,
|
|
"max_non_trigger_score": 0.228,
|
|
"score_gap": 0.417,
|
|
"threshold_margin": 0.165,
|
|
"positive_threshold_buffer": 0.165,
|
|
"negative_threshold_buffer": 0.252,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"current_calibration": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.82,
|
|
"mean_non_trigger_score": 0.059,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.645,
|
|
"max_non_trigger_score": 0.228,
|
|
"score_gap": 0.417,
|
|
"threshold_margin": 0.165,
|
|
"positive_threshold_buffer": 0.165,
|
|
"negative_threshold_buffer": 0.252,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"baseline_calibration": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.671,
|
|
"mean_non_trigger_score": 0.053,
|
|
"mean_near_neighbor_score": 0.003,
|
|
"min_positive_score": 0.55,
|
|
"max_non_trigger_score": 0.176,
|
|
"score_gap": 0.374,
|
|
"threshold_margin": 0.07,
|
|
"positive_threshold_buffer": 0.07,
|
|
"negative_threshold_buffer": 0.304,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "watch"
|
|
},
|
|
"winner_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"judge_blind_holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0,
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0,
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0,
|
|
"judge_summary": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"winner_calibration": null,
|
|
"current_calibration": null,
|
|
"baseline_calibration": null,
|
|
"winner_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"adversarial_holdout_non_regression": {
|
|
"winner": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"current": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"baseline": {
|
|
"false_positives": 0,
|
|
"false_negatives": 0,
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"near_neighbor_pass_rate": 1.0,
|
|
"should_not_trigger_pass_rate": 1.0
|
|
},
|
|
"winner_calibration": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.907,
|
|
"mean_non_trigger_score": 0.083,
|
|
"mean_near_neighbor_score": 0.139,
|
|
"min_positive_score": 0.815,
|
|
"max_non_trigger_score": 0.277,
|
|
"score_gap": 0.538,
|
|
"threshold_margin": 0.063,
|
|
"positive_threshold_buffer": 0.475,
|
|
"negative_threshold_buffer": 0.063,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "watch"
|
|
},
|
|
"current_calibration": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.907,
|
|
"mean_non_trigger_score": 0.083,
|
|
"mean_near_neighbor_score": 0.139,
|
|
"min_positive_score": 0.815,
|
|
"max_non_trigger_score": 0.277,
|
|
"score_gap": 0.538,
|
|
"threshold_margin": 0.063,
|
|
"positive_threshold_buffer": 0.475,
|
|
"negative_threshold_buffer": 0.063,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "watch"
|
|
},
|
|
"baseline_calibration": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.836,
|
|
"mean_non_trigger_score": 0.107,
|
|
"mean_near_neighbor_score": 0.101,
|
|
"min_positive_score": 0.682,
|
|
"max_non_trigger_score": 0.215,
|
|
"score_gap": 0.467,
|
|
"threshold_margin": 0.125,
|
|
"positive_threshold_buffer": 0.342,
|
|
"negative_threshold_buffer": 0.125,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"winner_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_single_update_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"current_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_single_update_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"baseline_family_health": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_single_update_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"summary": {
|
|
"winner_label": "Current",
|
|
"winner_tokens": 37,
|
|
"current_tokens": 37,
|
|
"winner_dev_total_errors": 0,
|
|
"current_dev_total_errors": 0,
|
|
"winner_holdout_total_errors": 1,
|
|
"current_holdout_total_errors": 1,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"current_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"current_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"current_adversarial_holdout_total_errors": 0,
|
|
"winner_judge_blind_agreement_rate": 1.0,
|
|
"winner_adversarial_risk_band": "watch",
|
|
"winner_adversarial_score_gap": 0.538,
|
|
"candidate_count": 5,
|
|
"baseline_tokens": 93,
|
|
"baseline_dev_total_errors": 0,
|
|
"baseline_holdout_total_errors": 1,
|
|
"baseline_blind_holdout_total_errors": 0,
|
|
"baseline_judge_blind_holdout_total_errors": 0,
|
|
"baseline_adversarial_holdout_total_errors": 0
|
|
}
|
|
}
|