YAO
31ce04c655
Merge the beta-ready Yao Meta Skill architecture, report, evidence gate, and release-boundary updates.\n\nRelease boundary: beta/public testing is allowed; formal world-class, fully reviewed, or superiority claims remain blocked until the pending evidence gates are accepted.
438 lines
14 KiB
JSON
438 lines
14 KiB
JSON
{
|
|
"targets": [
|
|
{
|
|
"name": "yao-meta-skill",
|
|
"winner_label": "Current",
|
|
"winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use for skill creation, reusable workflow packaging, skill improvement, evals, and team-ready distribution.",
|
|
"winner_tokens": 53,
|
|
"current_tokens": 53,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 0,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 0,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 0,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.908,
|
|
"mean_non_trigger_score": 0.003,
|
|
"mean_near_neighbor_score": 0.006,
|
|
"min_positive_score": 0.767,
|
|
"max_non_trigger_score": 0.023,
|
|
"score_gap": 0.744,
|
|
"threshold_margin": 0.307,
|
|
"positive_threshold_buffer": 0.437,
|
|
"negative_threshold_buffer": 0.307,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 0.669,
|
|
"mean_non_trigger_score": 0.116,
|
|
"mean_near_neighbor_score": 0.152,
|
|
"min_positive_score": 0.665,
|
|
"max_non_trigger_score": 0.305,
|
|
"score_gap": 0.36,
|
|
"threshold_margin": 0.025,
|
|
"positive_threshold_buffer": 0.335,
|
|
"negative_threshold_buffer": 0.025,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "tight"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.33,
|
|
"mean_positive_score": 1.0,
|
|
"mean_non_trigger_score": 0.0,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 1.0,
|
|
"max_non_trigger_score": 0.0,
|
|
"score_gap": 1.0,
|
|
"threshold_margin": 0.33,
|
|
"positive_threshold_buffer": 0.67,
|
|
"negative_threshold_buffer": 0.33,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.66,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 12,
|
|
"clean_family_count": 12,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "workflow_to_skill",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_operationalize",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_summary_translation_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true
|
|
},
|
|
{
|
|
"name": "team-frontend-review",
|
|
"winner_label": "Current",
|
|
"winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.",
|
|
"winner_tokens": 50,
|
|
"current_tokens": 50,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 0,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 0,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 0,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.685,
|
|
"mean_non_trigger_score": 0.089,
|
|
"mean_near_neighbor_score": 0.045,
|
|
"min_positive_score": 0.496,
|
|
"max_non_trigger_score": 0.211,
|
|
"score_gap": 0.285,
|
|
"threshold_margin": 0.016,
|
|
"positive_threshold_buffer": 0.016,
|
|
"negative_threshold_buffer": 0.269,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "tight"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.685,
|
|
"mean_non_trigger_score": 0.177,
|
|
"mean_near_neighbor_score": 0.226,
|
|
"min_positive_score": 0.504,
|
|
"max_non_trigger_score": 0.416,
|
|
"score_gap": 0.088,
|
|
"threshold_margin": 0.024,
|
|
"positive_threshold_buffer": 0.024,
|
|
"negative_threshold_buffer": 0.064,
|
|
"boundary_case_count": 2,
|
|
"boundary_case_rate": 0.333,
|
|
"risk_band": "tight"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.71,
|
|
"mean_non_trigger_score": 0.009,
|
|
"mean_near_neighbor_score": 0.018,
|
|
"min_positive_score": 0.544,
|
|
"max_non_trigger_score": 0.035,
|
|
"score_gap": 0.509,
|
|
"threshold_margin": 0.204,
|
|
"positive_threshold_buffer": 0.204,
|
|
"negative_threshold_buffer": 0.305,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.703,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "security_and_a11y",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_premerge",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_premerge",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_quality_gate_review",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true
|
|
},
|
|
{
|
|
"name": "governed-incident-command",
|
|
"winner_label": "Current",
|
|
"winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.",
|
|
"winner_tokens": 37,
|
|
"current_tokens": 37,
|
|
"winner_holdout_fp": 0,
|
|
"winner_holdout_fn": 1,
|
|
"current_holdout_fp": 0,
|
|
"current_holdout_fn": 1,
|
|
"baseline_holdout_fp": 0,
|
|
"baseline_holdout_fn": 1,
|
|
"winner_blind_holdout_fp": 0,
|
|
"winner_blind_holdout_fn": 0,
|
|
"current_blind_holdout_fp": 0,
|
|
"current_blind_holdout_fn": 0,
|
|
"baseline_blind_holdout_fp": 0,
|
|
"baseline_blind_holdout_fn": 0,
|
|
"winner_blind_holdout_total_errors": 0,
|
|
"winner_judge_blind_holdout_fp": 0,
|
|
"winner_judge_blind_holdout_fn": 0,
|
|
"current_judge_blind_holdout_fp": 0,
|
|
"current_judge_blind_holdout_fn": 0,
|
|
"baseline_judge_blind_holdout_fp": 0,
|
|
"baseline_judge_blind_holdout_fn": 0,
|
|
"winner_judge_blind_holdout_total_errors": 0,
|
|
"winner_adversarial_holdout_fp": 0,
|
|
"winner_adversarial_holdout_fn": 0,
|
|
"current_adversarial_holdout_fp": 0,
|
|
"current_adversarial_holdout_fn": 0,
|
|
"baseline_adversarial_holdout_fp": 0,
|
|
"baseline_adversarial_holdout_fn": 0,
|
|
"winner_adversarial_holdout_total_errors": 0,
|
|
"calibration": {
|
|
"holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.45,
|
|
"mean_non_trigger_score": 0.019,
|
|
"mean_near_neighbor_score": 0.037,
|
|
"min_positive_score": 0.08,
|
|
"max_non_trigger_score": 0.057,
|
|
"score_gap": 0.023,
|
|
"threshold_margin": -0.4,
|
|
"positive_threshold_buffer": -0.4,
|
|
"negative_threshold_buffer": 0.423,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "overlap"
|
|
},
|
|
"blind_holdout": {
|
|
"threshold": 0.48,
|
|
"mean_positive_score": 0.82,
|
|
"mean_non_trigger_score": 0.059,
|
|
"mean_near_neighbor_score": 0.0,
|
|
"min_positive_score": 0.645,
|
|
"max_non_trigger_score": 0.228,
|
|
"score_gap": 0.417,
|
|
"threshold_margin": 0.165,
|
|
"positive_threshold_buffer": 0.165,
|
|
"negative_threshold_buffer": 0.252,
|
|
"boundary_case_count": 0,
|
|
"boundary_case_rate": 0.0,
|
|
"risk_band": "healthy"
|
|
},
|
|
"adversarial_holdout": {
|
|
"threshold": 0.34,
|
|
"mean_positive_score": 0.907,
|
|
"mean_non_trigger_score": 0.083,
|
|
"mean_near_neighbor_score": 0.139,
|
|
"min_positive_score": 0.815,
|
|
"max_non_trigger_score": 0.277,
|
|
"score_gap": 0.538,
|
|
"threshold_margin": 0.063,
|
|
"positive_threshold_buffer": 0.475,
|
|
"negative_threshold_buffer": 0.063,
|
|
"boundary_case_count": 1,
|
|
"boundary_case_rate": 0.167,
|
|
"risk_band": "watch"
|
|
}
|
|
},
|
|
"judge_blind": {
|
|
"winner": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"current": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
},
|
|
"baseline": {
|
|
"agreement_rate": 1.0,
|
|
"mean_confidence": 0.677,
|
|
"rubric_version": "blind-v1"
|
|
}
|
|
},
|
|
"family_health": {
|
|
"holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 5,
|
|
"failing_families": [
|
|
{
|
|
"family": "packet_assembly",
|
|
"errors": 1,
|
|
"pass_rate": 0.0
|
|
}
|
|
],
|
|
"weakest_family": {
|
|
"family": "packet_assembly",
|
|
"pass_rate": 0.0,
|
|
"errors": 1
|
|
}
|
|
},
|
|
"blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"judge_blind_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "blind_summary_only",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
},
|
|
"adversarial_holdout": {
|
|
"family_count": 6,
|
|
"clean_family_count": 6,
|
|
"failing_families": [],
|
|
"weakest_family": {
|
|
"family": "adversarial_single_update_collision",
|
|
"pass_rate": 1.0,
|
|
"errors": 0
|
|
}
|
|
}
|
|
},
|
|
"drift_note": "blind, judge-backed blind, adversarial, and calibration gates active",
|
|
"ok": true
|
|
}
|
|
],
|
|
"ok": true
|
|
}
|