yaojingang
438 lines
14 KiB
JSON
438 lines
14 KiB
JSON
{
|
|
"targets": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"winner_label": "Current",
|
|
"winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.",
|
|
"winner_tokens": 65,
|
|
"current_tokens": 65,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 0,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 0,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 0,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.672,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.468,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.468,
|
|
"threshold_margin": 0.138,
|
|
"positive_threshold_buffer": 0.138,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.399,
|
|
"mean_non_trigger_score": 0.013,
|
|
"mean_near_neighbor_score": 0.013,
|
|
"min_positive_score": 0.392,
|
|
"max_non_trigger_score": 0.026,
|
|
"score_gap": 0.366,
|
|
"threshold_margin": 0.062,
|
|
"positive_threshold_buffer": 0.062,
|
|
"negative_threshold_buffer": 0.304,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "watch"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.838,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.834,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 0.834,
|
|
"threshold_margin": 0.33,
|
|
"positive_threshold_buffer": 0.504,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 12,
|
|
"clean_family_count": 12,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "workflow_to_skill",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_summary_translation_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"winner_label": "Current",
|
|
"winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
|
|
"winner_tokens": 50,
|
|
"current_tokens": 50,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 0,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 0,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 0,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.681,
|
|
"mean_non_trigger_score": 0.088,
|
|
"mean_near_neighbor_score": 0.042,
|
|
"min_positive_score": 0.49,
|
|
"max_non_trigger_score": 0.211,
|
|
"score_gap": 0.279,
|
|
"threshold_margin": 0.01,
|
|
"positive_threshold_buffer": 0.01,
|
|
"negative_threshold_buffer": 0.269,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "tight"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.681,
|
|
"mean_non_trigger_score": 0.177,
|
|
"mean_near_neighbor_score": 0.226,
|
|
"min_positive_score": 0.504,
|
|
"max_non_trigger_score": 0.416,
|
|
"score_gap": 0.088,
|
|
"threshold_margin": 0.024,
|
|
"positive_threshold_buffer": 0.024,
|
|
"negative_threshold_buffer": 0.064,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "tight"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.71,
|
|
"mean_non_trigger_score": 0.009,
|
|
"mean_near_neighbor_score": 0.018,
|
|
"min_positive_score": 0.544,
|
|
"max_non_trigger_score": 0.035,
|
|
"score_gap": 0.509,
|
|
"threshold_margin": 0.204,
|
|
"positive_threshold_buffer": 0.204,
|
|
"negative_threshold_buffer": 0.305,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "security_and_a11y",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_premerge",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_premerge",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_quality_gate_review",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"winner_label": "Current",
|
|
"winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"winner_tokens": 37,
|
|
"current_tokens": 37,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 1,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 1,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 2,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 1,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.367,
|
|
"mean_non_trigger_score": 0.019,
|
|
"mean_near_neighbor_score": 0.038,
|
|
"min_positive_score": 0.08,
|
|
"max_non_trigger_score": 0.058,
|
|
"score_gap": 0.022,
|
|
"threshold_margin": -0.4,
|
|
"positive_threshold_buffer": -0.4,
|
|
"negative_threshold_buffer": 0.422,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "overlap"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.899,
|
|
"mean_non_trigger_score": 0.168,
|
|
"mean_near_neighbor_score": 0.138,
|
|
"min_positive_score": 0.798,
|
|
"max_non_trigger_score": 0.386,
|
|
"score_gap": 0.412,
|
|
"threshold_margin": 0.094,
|
|
"positive_threshold_buffer": 0.318,
|
|
"negative_threshold_buffer": 0.094,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "healthy"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.827,
|
|
"mean_non_trigger_score": 0.027,
|
|
"mean_near_neighbor_score": 0.011,
|
|
"min_positive_score": 0.654,
|
|
"max_non_trigger_score": 0.056,
|
|
"score_gap": 0.598,
|
|
"threshold_margin": 0.284,
|
|
"positive_threshold_buffer": 0.314,
|
|
"negative_threshold_buffer": 0.284,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.657,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.657,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.69,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 5,
|
|
"failing_families": [
|
|
{
|
|
"family": "packet_assembly",
|
|
"errors": 1,
|
|
"pass_rate": 0.0
|
|
}
|
|
],
|
|
"weakest_family": {
|
|
"family": "packet_assembly",
|
|
"pass_rate": 0.0,
|
|
"errors": 1
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_single_update_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true
|
|
}
|
|
],
|
|
"ok": true
|
|
}
|