Harden world-class release gates
This commit is contained in:
+782
-585
File diff suppressed because it is too large
Load Diff
+12
-12
@@ -14,18 +14,18 @@ Build governed incident command packets. Use when asked to standardize incident
|
||||
|
||||
| Candidate | Tokens | Dev FP | Dev FN | Dev Near | Holdout FP | Holdout FN |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| `Current` | 37 | 0 | 1 | 1.0 | 0 | 1 |
|
||||
| `Guardrail` | 51 | 0 | 1 | 1.0 | 1 | 1 |
|
||||
| `Balanced` | 54 | 0 | 1 | 1.0 | 0 | 1 |
|
||||
| `Boundary` | 78 | 0 | 1 | 1.0 | 0 | 1 |
|
||||
| `Artifact Aware` | 78 | 0 | 1 | 1.0 | 0 | 1 |
|
||||
| `Current` | 37 | 0 | 0 | 1.0 | 0 | 1 |
|
||||
| `Balanced` | 54 | 0 | 0 | 1.0 | 0 | 1 |
|
||||
| `Boundary` | 78 | 0 | 0 | 1.0 | 0 | 1 |
|
||||
| `Artifact Aware` | 78 | 0 | 0 | 1.0 | 0 | 1 |
|
||||
| `Guardrail` | 51 | 0 | 1 | 1.0 | 0 | 1 |
|
||||
|
||||
## Acceptance Gates
|
||||
|
||||
| Gate | Winner FP | Winner FN | Current FP | Current FN | Baseline FP | Baseline FN |
|
||||
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
|
||||
| Holdout | 0 | 1 | 0 | 1 | 0 | 2 |
|
||||
| Blind Holdout | 0 | 0 | 0 | 0 | 0 | 1 |
|
||||
| Holdout | 0 | 1 | 0 | 1 | 0 | 1 |
|
||||
| Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 |
|
||||
| Judge Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 |
|
||||
| Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 |
|
||||
|
||||
@@ -34,21 +34,21 @@ Build governed incident command packets. Use when asked to standardize incident
|
||||
| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |
|
||||
| --- | ---: | --- | ---: | ---: | ---: |
|
||||
| Holdout | 0.023 | overlap | 0.0 | 0.023 | 0.101 |
|
||||
| Blind Holdout | 0.418 | healthy | 0.167 | 0.418 | 0.063 |
|
||||
| Adversarial Holdout | 0.598 | healthy | 0.0 | 0.598 | 0.172 |
|
||||
| Blind Holdout | 0.417 | healthy | 0.0 | 0.417 | 0.374 |
|
||||
| Adversarial Holdout | 0.538 | watch | 0.167 | 0.538 | 0.467 |
|
||||
|
||||
## Judge Blind Summary
|
||||
|
||||
| Gate | Winner Agreement | Winner Mean Confidence | Current Agreement | Baseline Agreement |
|
||||
| --- | ---: | ---: | ---: | ---: |
|
||||
| Judge Blind Holdout | 1.0 | 0.657 | 1.0 | 1.0 |
|
||||
| Judge Blind Holdout | 1.0 | 0.677 | 1.0 | 1.0 |
|
||||
|
||||
## Family Health
|
||||
|
||||
| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| Holdout | 5/6 | packet_assembly (1 errors) | 5/6 | 4/6 |
|
||||
| Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 5/6 |
|
||||
| Holdout | 5/6 | packet_assembly (1 errors) | 5/6 | 5/6 |
|
||||
| Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 6/6 |
|
||||
| Judge Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 6/6 |
|
||||
| Adversarial Holdout | 6/6 | adversarial_single_update_collision (0 errors) | 6/6 | 6/6 |
|
||||
|
||||
|
||||
@@ -37,9 +37,12 @@
|
||||
"weight": 0.32,
|
||||
"phrases": [
|
||||
"incident command packet",
|
||||
"incident command packets",
|
||||
"governed incident packet",
|
||||
"incident packet assembly",
|
||||
"incident review skill"
|
||||
"incident review skill",
|
||||
"incident review",
|
||||
"standardize incident review"
|
||||
]
|
||||
},
|
||||
"incident_inputs": {
|
||||
@@ -59,6 +62,8 @@
|
||||
"severity assessment",
|
||||
"severity",
|
||||
"impact scope",
|
||||
"incident review",
|
||||
"outage review",
|
||||
"sev"
|
||||
]
|
||||
},
|
||||
@@ -68,6 +73,8 @@
|
||||
"stakeholder update",
|
||||
"incident communication",
|
||||
"owner",
|
||||
"owners",
|
||||
"owner actions",
|
||||
"action ownership",
|
||||
"next actions"
|
||||
]
|
||||
@@ -76,6 +83,9 @@
|
||||
"weight": 0.1,
|
||||
"phrases": [
|
||||
"governed",
|
||||
"governed workflow",
|
||||
"reusable governed workflow",
|
||||
"standardize",
|
||||
"reviewed",
|
||||
"audited",
|
||||
"maintained",
|
||||
@@ -128,9 +138,13 @@
|
||||
"exclusive": true,
|
||||
"phrases": [
|
||||
"brainstorm",
|
||||
"sketch a future",
|
||||
"for later",
|
||||
"possible response ideas",
|
||||
"improve incident handling",
|
||||
"ideas for later"
|
||||
"ideas for later",
|
||||
"do not package",
|
||||
"do not package a reusable workflow"
|
||||
]
|
||||
},
|
||||
"summary_only": {
|
||||
|
||||
Reference in New Issue
Block a user