yaojingang
793 lines
29 KiB
JSON
793 lines
29 KiB
JSON
{
|
|
"ok": true,
|
|
"summary": {
|
|
"promote": 0,
|
|
"keep_current": 3,
|
|
"blocked": 0
|
|
},
|
|
"targets": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"winner_label": "Current",
|
|
"winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
|
|
"winner_tokens": 65,
|
|
"current_tokens": 65,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 0,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 0,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 0,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.673,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.468,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.468,
|
|
"threshold_margin": 0.138,
|
|
"positive_threshold_buffer": 0.138,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.399,
|
|
"mean_non_trigger_score": 0.013,
|
|
"mean_near_neighbor_score": 0.013,
|
|
"min_positive_score": 0.392,
|
|
"max_non_trigger_score": 0.028,
|
|
"score_gap": 0.364,
|
|
"threshold_margin": 0.062,
|
|
"positive_threshold_buffer": 0.062,
|
|
"negative_threshold_buffer": 0.302,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "watch"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.838,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.834,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.834,
|
|
"threshold_margin": 0.33,
|
|
"positive_threshold_buffer": 0.504,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 12,
|
|
"clean_family_count": 12,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "workflow_to_skill",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_summary_translation_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true,
|
|
"promotion": {
|
|
"decision": "keep_current",
|
|
"winner_changed": false,
|
|
"all_gates_pass": true,
|
|
"route_clean": true,
|
|
"judge_agreement_ok": true,
|
|
"causes": [
|
|
"no_candidate_outperformed_current"
|
|
],
|
|
"next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.",
|
|
"gate_status": {
|
|
"visible_holdout_non_regression": true,
|
|
"blind_holdout_non_regression": true,
|
|
"judge_blind_non_regression": true,
|
|
"judge_blind_agreement": true,
|
|
"adversarial_non_regression": true,
|
|
"adversarial_risk_ok": true,
|
|
"route_confusion_clean": true,
|
|
"family_stability": true
|
|
},
|
|
"route_metrics": {
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"average_margin": 0.611,
|
|
"misroute_count": 0,
|
|
"ambiguous_case_count": 0
|
|
}
|
|
},
|
|
"registry": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"role": "baseline",
|
|
"label": "Baseline",
|
|
"ranking_state": "reference",
|
|
"promotion_state": "reference",
|
|
"description": "Create and improve agent skills.",
|
|
"tokens": 8,
|
|
"dev_errors": 1,
|
|
"holdout_errors": 0
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"role": "current",
|
|
"label": "Current",
|
|
"ranking_state": "selected_by_dev",
|
|
"promotion_state": "kept_current",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
|
|
"tokens": 65,
|
|
"dev_errors": 0,
|
|
"holdout_errors": 0,
|
|
"reason_tags": []
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"role": "candidate",
|
|
"label": "Minimal",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Create, refactor, evaluate, and package agent skills. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
|
|
"tokens": 41,
|
|
"dev_errors": 2,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"weaker_dev_fit"
|
|
]
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"role": "candidate",
|
|
"label": "Guardrail",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
|
|
"tokens": 56,
|
|
"dev_errors": 2,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"weaker_dev_fit"
|
|
]
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"role": "candidate",
|
|
"label": "Balanced",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
|
|
"tokens": 60,
|
|
"dev_errors": 2,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"weaker_dev_fit"
|
|
]
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"role": "candidate",
|
|
"label": "Artifact Aware",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Trigger when requests mention workflows, prompts, transcripts, or docs and the job is to create a skill, turn a repeated process into a reusable skill, or improve an existing skill.",
|
|
"tokens": 77,
|
|
"dev_errors": 2,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"weaker_dev_fit",
|
|
"longer_without_gain"
|
|
]
|
|
},
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"role": "candidate",
|
|
"label": "Boundary",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, notes, 流程, 提示词, 对话记录, 文档, or 笔记. Use when asked to create a skill, turn a repeated process into a reusable skill, or improve an existing skill. Do not use for summary-only requests, translation-only requests, or plain documentation tasks.",
|
|
"tokens": 83,
|
|
"dev_errors": 2,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"weaker_dev_fit",
|
|
"longer_without_gain"
|
|
]
|
|
}
|
|
],
|
|
"decision": "keep_current"
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"winner_label": "Current",
|
|
"winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
|
|
"winner_tokens": 50,
|
|
"current_tokens": 50,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 0,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 0,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 0,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.685,
|
|
"mean_non_trigger_score": 0.089,
|
|
"mean_near_neighbor_score": 0.045,
|
|
"min_positive_score": 0.496,
|
|
"max_non_trigger_score": 0.211,
|
|
"score_gap": 0.285,
|
|
"threshold_margin": 0.016,
|
|
"positive_threshold_buffer": 0.016,
|
|
"negative_threshold_buffer": 0.269,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "tight"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.685,
|
|
"mean_non_trigger_score": 0.177,
|
|
"mean_near_neighbor_score": 0.226,
|
|
"min_positive_score": 0.504,
|
|
"max_non_trigger_score": 0.416,
|
|
"score_gap": 0.088,
|
|
"threshold_margin": 0.024,
|
|
"positive_threshold_buffer": 0.024,
|
|
"negative_threshold_buffer": 0.064,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "tight"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.71,
|
|
"mean_non_trigger_score": 0.009,
|
|
"mean_near_neighbor_score": 0.018,
|
|
"min_positive_score": 0.544,
|
|
"max_non_trigger_score": 0.035,
|
|
"score_gap": 0.509,
|
|
"threshold_margin": 0.204,
|
|
"positive_threshold_buffer": 0.204,
|
|
"negative_threshold_buffer": 0.305,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "security_and_a11y",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_premerge",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_premerge",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_quality_gate_review",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true,
|
|
"promotion": {
|
|
"decision": "keep_current",
|
|
"winner_changed": false,
|
|
"all_gates_pass": true,
|
|
"route_clean": true,
|
|
"judge_agreement_ok": true,
|
|
"causes": [
|
|
"no_candidate_outperformed_current",
|
|
"current_holdout_risk"
|
|
],
|
|
"next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.",
|
|
"gate_status": {
|
|
"visible_holdout_non_regression": true,
|
|
"blind_holdout_non_regression": true,
|
|
"judge_blind_non_regression": true,
|
|
"judge_blind_agreement": true,
|
|
"adversarial_non_regression": true,
|
|
"adversarial_risk_ok": true,
|
|
"route_confusion_clean": true,
|
|
"family_stability": true
|
|
},
|
|
"route_metrics": {
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"average_margin": 0.805,
|
|
"misroute_count": 0,
|
|
"ambiguous_case_count": 0
|
|
}
|
|
},
|
|
"registry": [
|
|
{
|
|
"name": "team-frontend-review",
|
|
"role": "baseline",
|
|
"label": "Baseline",
|
|
"ranking_state": "reference",
|
|
"promotion_state": "reference",
|
|
"description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.",
|
|
"tokens": 52,
|
|
"dev_errors": 3,
|
|
"holdout_errors": 0
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"role": "current",
|
|
"label": "Current",
|
|
"ranking_state": "selected_by_dev",
|
|
"promotion_state": "kept_current",
|
|
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
|
|
"tokens": 50,
|
|
"dev_errors": 3,
|
|
"holdout_errors": 0,
|
|
"reason_tags": []
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"role": "candidate",
|
|
"label": "Guardrail",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.",
|
|
"tokens": 62,
|
|
"dev_errors": 3,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"longer_without_gain"
|
|
]
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"role": "candidate",
|
|
"label": "Balanced",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
|
|
"tokens": 64,
|
|
"dev_errors": 3,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"longer_without_gain"
|
|
]
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"role": "candidate",
|
|
"label": "Artifact Aware",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Trigger when requests mention React components, forms, frontend diffs, or UI code and the job is to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
|
|
"tokens": 84,
|
|
"dev_errors": 3,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"longer_without_gain"
|
|
]
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"role": "candidate",
|
|
"label": "Boundary",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.",
|
|
"tokens": 90,
|
|
"dev_errors": 3,
|
|
"holdout_errors": 0,
|
|
"reason_tags": [
|
|
"longer_without_gain"
|
|
]
|
|
}
|
|
],
|
|
"decision": "keep_current"
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"winner_label": "Current",
|
|
"winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"winner_tokens": 37,
|
|
"current_tokens": 37,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 1,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 1,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 2,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 1,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.37,
|
|
"mean_non_trigger_score": 0.019,
|
|
"mean_near_neighbor_score": 0.037,
|
|
"min_positive_score": 0.08,
|
|
"max_non_trigger_score": 0.057,
|
|
"score_gap": 0.023,
|
|
"threshold_margin": -0.4,
|
|
"positive_threshold_buffer": -0.4,
|
|
"negative_threshold_buffer": 0.423,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "overlap"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.902,
|
|
"mean_non_trigger_score": 0.168,
|
|
"mean_near_neighbor_score": 0.138,
|
|
"min_positive_score": 0.804,
|
|
"max_non_trigger_score": 0.386,
|
|
"score_gap": 0.418,
|
|
"threshold_margin": 0.094,
|
|
"positive_threshold_buffer": 0.324,
|
|
"negative_threshold_buffer": 0.094,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "healthy"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.827,
|
|
"mean_non_trigger_score": 0.027,
|
|
"mean_near_neighbor_score": 0.011,
|
|
"min_positive_score": 0.654,
|
|
"max_non_trigger_score": 0.056,
|
|
"score_gap": 0.598,
|
|
"threshold_margin": 0.284,
|
|
"positive_threshold_buffer": 0.314,
|
|
"negative_threshold_buffer": 0.284,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.657,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.657,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.69,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 5,
|
|
"failing_families": [
|
|
{
|
|
"family": "packet_assembly",
|
|
"errors": 1,
|
|
"pass_rate": 0.0
|
|
}
|
|
],
|
|
"weakest_family": {
|
|
"family": "packet_assembly",
|
|
"pass_rate": 0.0,
|
|
"errors": 1
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_single_update_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true,
|
|
"promotion": {
|
|
"decision": "keep_current",
|
|
"winner_changed": false,
|
|
"all_gates_pass": true,
|
|
"route_clean": true,
|
|
"judge_agreement_ok": true,
|
|
"causes": [
|
|
"no_candidate_outperformed_current",
|
|
"current_holdout_gap_present",
|
|
"current_holdout_risk"
|
|
],
|
|
"next_action": "Keep the current description and open a new candidate only when fresh route evidence appears.",
|
|
"gate_status": {
|
|
"visible_holdout_non_regression": true,
|
|
"blind_holdout_non_regression": true,
|
|
"judge_blind_non_regression": true,
|
|
"judge_blind_agreement": true,
|
|
"adversarial_non_regression": true,
|
|
"adversarial_risk_ok": true,
|
|
"route_confusion_clean": true,
|
|
"family_stability": true
|
|
},
|
|
"route_metrics": {
|
|
"precision": 1.0,
|
|
"recall": 1.0,
|
|
"average_margin": 0.659,
|
|
"misroute_count": 0,
|
|
"ambiguous_case_count": 0
|
|
}
|
|
},
|
|
"registry": [
|
|
{
|
|
"name": "governed-incident-command",
|
|
"role": "baseline",
|
|
"label": "Baseline",
|
|
"ranking_state": "reference",
|
|
"promotion_state": "reference",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, and stakeholder requirements. Use when asked to standardize incident review, severity assessment, incident communication, or incident packet assembly into one reusable operational skill. Do not use for generic debugging, one-off incident explanations, or brainstorming possible response ideas.",
|
|
"tokens": 93,
|
|
"dev_errors": 1,
|
|
"holdout_errors": 2
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"role": "current",
|
|
"label": "Current",
|
|
"ranking_state": "selected_by_dev",
|
|
"promotion_state": "kept_current",
|
|
"description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"tokens": 37,
|
|
"dev_errors": 1,
|
|
"holdout_errors": 1,
|
|
"reason_tags": []
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"role": "candidate",
|
|
"label": "Guardrail",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
|
|
"tokens": 51,
|
|
"dev_errors": 1,
|
|
"holdout_errors": 2,
|
|
"reason_tags": [
|
|
"visible_holdout_regression",
|
|
"longer_without_gain"
|
|
]
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"role": "candidate",
|
|
"label": "Balanced",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"tokens": 54,
|
|
"dev_errors": 1,
|
|
"holdout_errors": 1,
|
|
"reason_tags": [
|
|
"longer_without_gain"
|
|
]
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"role": "candidate",
|
|
"label": "Boundary",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Use when asked to standardize incident review, run severity assessment, or assemble incident communication. Do not use for generic debugging, one-off incident explanations, or drafting a single update.",
|
|
"tokens": 78,
|
|
"dev_errors": 1,
|
|
"holdout_errors": 1,
|
|
"reason_tags": [
|
|
"longer_without_gain"
|
|
]
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"role": "candidate",
|
|
"label": "Artifact Aware",
|
|
"ranking_state": "not_selected",
|
|
"promotion_state": "blocked",
|
|
"description": "Build governed incident command packets from alerts, incident notes, transcripts, or stakeholder requirements. Trigger when requests mention alerts, incident notes, transcripts, or stakeholder requirements and the job is to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"tokens": 78,
|
|
"dev_errors": 1,
|
|
"holdout_errors": 1,
|
|
"reason_tags": [
|
|
"longer_without_gain"
|
|
]
|
|
}
|
|
],
|
|
"decision": "keep_current"
|
|
}
|
|
]
|
|
}
|