{ "current_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", "current_candidate": { "id": "current", "label": "Current", "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", "strategy": "current", "estimated_tokens": 50, "dev": { "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 3, -1.0, -1.0, -1.0, -0.25, 50 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, "baseline": { "description": "Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.", "estimated_tokens": 52, "dev": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "bucket_stats": { "should_trigger": { "total": 4, "passed": 1, "pass_rate": 0.25 }, "should_not_trigger": { "total": 4, "passed": 4, "pass_rate": 1.0 }, "near_neighbor": { "total": 3, "passed": 3, "pass_rate": 1.0 } }, "family_stats": { "premerge_review": { "total": 2, "passed": 0, "false_positives": 0, "false_negatives": 2, "pass_rate": 0.0 }, "state_review": { "total": 1, "passed": 0, "false_positives": 0, "false_negatives": 1, "pass_rate": 0.0 }, "security_and_a11y": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "backend_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [ { "bucket": "should_trigger", "family": "premerge_review", "prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.", "score": 0.316, "reason": "false_negative", "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "matched_negative_concepts": [] }, { "bucket": "should_trigger", "family": "state_review", "prompt": "Do a frontend review on these UI components and check loading, error, and empty states.", "score": 0.424, "reason": "false_negative", "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "matched_negative_concepts": [] }, { "bucket": "should_trigger", "family": "premerge_review", "prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.", "score": 0.188, "reason": "false_negative", "matched_desired_concepts": [ "frontend_scope", "ux_regression" ], "matched_negative_concepts": [] } ], "results": { "should_trigger": [ { "prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.", "family": "premerge_review", "score": 0.316, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.692, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "react", "frontend", "form" ] }, "negative": {} } } }, { "prompt": "Do a frontend review on these UI components and check loading, error, and empty states.", "family": "state_review", "score": 0.424, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.429, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend", "ui" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Audit this signup form for a11y, unsafe rendering, and UX regressions.", "family": "security_and_a11y", "score": 0.483, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.455, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "form" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.", "family": "premerge_review", "score": 0.188, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.12, "support_score": 0, "lexical_support": 0.467, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "react", "ui" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a React login page with loading and error states.", "family": "build_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "build_ui" ], "exclusive_negative_concepts": [ "build_ui" ], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": { "build_ui": [ "build a react" ] } } } }, { "prompt": "Explain how accessibility works in React forms.", "family": "explain_only", "score": 0.295, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.429, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Review this backend endpoint and database migration.", "family": "backend_only", "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": {} } } }, { "prompt": "Translate these frontend review comments into Japanese.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "translate_only" ], "exclusive_negative_concepts": [ "translate_only" ], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.35, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": { "translate_only": [ "translate", "into japanese" ] } } } } ], "near_neighbor": [ { "prompt": "Brainstorm ways to improve this dashboard UX, but do not review code.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "brainstorm_only": [ "brainstorm" ] } } } }, { "prompt": "Write a frontend review checklist without reviewing the code itself.", "family": "checklist_only", "score": 0.414, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Fix the failing validation bug in this form component.", "family": "fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ] }, "negative": {} } } } ] } }, "holdout": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "premerge_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "security_and_a11y": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Give this React modal a pre-merge frontend review for a11y, risky rendering, and missing states.", "family": "premerge_review", "score": 0.869, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "state_coverage", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.82, "support_score": 0, "lexical_support": 0.688, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react", "frontend" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } }, { "prompt": "Review this UI diff for accessibility gaps, unsafe form handling, and UX regressions.", "family": "security_and_a11y", "score": 0.489, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.538, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "ui", "form" ], "ui_security": [ "form handling" ] }, "negative": {} } }, "boundary_case": true } ], "should_not_trigger": [ { "prompt": "Create a polished React dashboard page from scratch.", "family": "build_only", "score": 0.047, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.125, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Explain keyboard accessibility rules for beginners.", "family": "explain_only", "score": 0.211, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.2, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Discuss possible UI improvements for this checkout flow, but do not inspect the code yet.", "family": "brainstorm_only", "score": 0.053, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "ui" ] }, "negative": {} } } }, { "prompt": "Repair this broken input validation behavior in the form component.", "family": "fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ] }, "negative": {} } } } ] } } }, "winner": { "id": "current", "label": "Current", "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", "strategy": "current", "estimated_tokens": 50, "dev": { "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 3, -1.0, -1.0, -1.0, -0.25, 50 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, "winner_dev_report": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "bucket_stats": { "should_trigger": { "total": 4, "passed": 1, "pass_rate": 0.25 }, "should_not_trigger": { "total": 4, "passed": 4, "pass_rate": 1.0 }, "near_neighbor": { "total": 3, "passed": 3, "pass_rate": 1.0 } }, "family_stats": { "premerge_review": { "total": 2, "passed": 0, "false_positives": 0, "false_negatives": 2, "pass_rate": 0.0 }, "state_review": { "total": 1, "passed": 0, "false_positives": 0, "false_negatives": 1, "pass_rate": 0.0 }, "security_and_a11y": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "backend_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [ { "bucket": "should_trigger", "family": "premerge_review", "prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.", "score": 0.304, "reason": "false_negative", "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "matched_negative_concepts": [] }, { "bucket": "should_trigger", "family": "state_review", "prompt": "Do a frontend review on these UI components and check loading, error, and empty states.", "score": 0.435, "reason": "false_negative", "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "matched_negative_concepts": [] }, { "bucket": "should_trigger", "family": "premerge_review", "prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.", "score": 0.188, "reason": "false_negative", "matched_desired_concepts": [ "frontend_scope", "ux_regression" ], "matched_negative_concepts": [] } ], "results": { "should_trigger": [ { "prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.", "family": "premerge_review", "score": 0.304, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.538, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "react", "frontend", "form" ] }, "negative": {} } } }, { "prompt": "Do a frontend review on these UI components and check loading, error, and empty states.", "family": "state_review", "score": 0.435, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.571, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend", "ui" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Audit this signup form for a11y, unsafe rendering, and UX regressions.", "family": "security_and_a11y", "score": 0.497, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.636, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "form" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.", "family": "premerge_review", "score": 0.188, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.12, "support_score": 0, "lexical_support": 0.467, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "react", "ui" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a React login page with loading and error states.", "family": "build_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "build_ui" ], "exclusive_negative_concepts": [ "build_ui" ], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": { "build_ui": [ "build a react" ] } } } }, { "prompt": "Explain how accessibility works in React forms.", "family": "explain_only", "score": 0.284, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Review this backend endpoint and database migration.", "family": "backend_only", "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": {} } } }, { "prompt": "Translate these frontend review comments into Japanese.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "translate_only" ], "exclusive_negative_concepts": [ "translate_only" ], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.35, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": { "translate_only": [ "translate", "into japanese" ] } } } } ], "near_neighbor": [ { "prompt": "Brainstorm ways to improve this dashboard UX, but do not review code.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "brainstorm_only": [ "brainstorm" ] } } } }, { "prompt": "Write a frontend review checklist without reviewing the code itself.", "family": "checklist_only", "score": 0.422, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Fix the failing validation bug in this form component.", "family": "fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ] }, "negative": {} } } } ] } }, "winner_holdout_report": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "premerge_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "security_and_a11y": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Give this React modal a pre-merge frontend review for a11y, risky rendering, and missing states.", "family": "premerge_review", "score": 0.874, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "state_coverage", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.82, "support_score": 0, "lexical_support": 0.75, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react", "frontend" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } }, { "prompt": "Review this UI diff for accessibility gaps, unsafe form handling, and UX regressions.", "family": "security_and_a11y", "score": 0.496, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.615, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "ui", "form" ], "ui_security": [ "form handling" ] }, "negative": {} } }, "boundary_case": true } ], "should_not_trigger": [ { "prompt": "Create a polished React dashboard page from scratch.", "family": "build_only", "score": 0.057, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Explain keyboard accessibility rules for beginners.", "family": "explain_only", "score": 0.211, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.2, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Discuss possible UI improvements for this checkout flow, but do not inspect the code yet.", "family": "brainstorm_only", "score": 0.053, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "ui" ] }, "negative": {} } } }, { "prompt": "Repair this broken input validation behavior in the form component.", "family": "fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ] }, "negative": {} } } } ] } }, "current_dev_report": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "bucket_stats": { "should_trigger": { "total": 4, "passed": 1, "pass_rate": 0.25 }, "should_not_trigger": { "total": 4, "passed": 4, "pass_rate": 1.0 }, "near_neighbor": { "total": 3, "passed": 3, "pass_rate": 1.0 } }, "family_stats": { "premerge_review": { "total": 2, "passed": 0, "false_positives": 0, "false_negatives": 2, "pass_rate": 0.0 }, "state_review": { "total": 1, "passed": 0, "false_positives": 0, "false_negatives": 1, "pass_rate": 0.0 }, "security_and_a11y": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "backend_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "translate_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [ { "bucket": "should_trigger", "family": "premerge_review", "prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.", "score": 0.304, "reason": "false_negative", "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "matched_negative_concepts": [] }, { "bucket": "should_trigger", "family": "state_review", "prompt": "Do a frontend review on these UI components and check loading, error, and empty states.", "score": 0.435, "reason": "false_negative", "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "matched_negative_concepts": [] }, { "bucket": "should_trigger", "family": "premerge_review", "prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.", "score": 0.188, "reason": "false_negative", "matched_desired_concepts": [ "frontend_scope", "ux_regression" ], "matched_negative_concepts": [] } ], "results": { "should_trigger": [ { "prompt": "Review this React checkout form for accessibility and risky frontend issues before merge.", "family": "premerge_review", "score": 0.304, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.538, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "react", "frontend", "form" ] }, "negative": {} } } }, { "prompt": "Do a frontend review on these UI components and check loading, error, and empty states.", "family": "state_review", "score": 0.435, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.571, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend", "ui" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Audit this signup form for a11y, unsafe rendering, and UX regressions.", "family": "security_and_a11y", "score": 0.497, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.636, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "form" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Review this React diff with a pre-merge mindset and call out UI quality risks.", "family": "premerge_review", "score": 0.188, "predicted_trigger": false, "expected_trigger": true, "passed": false, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.12, "support_score": 0, "lexical_support": 0.467, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "react", "ui" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a React login page with loading and error states.", "family": "build_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "build_ui" ], "exclusive_negative_concepts": [ "build_ui" ], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": { "build_ui": [ "build a react" ] } } } }, { "prompt": "Explain how accessibility works in React forms.", "family": "explain_only", "score": 0.284, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Review this backend endpoint and database migration.", "family": "backend_only", "score": 0.023, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": {} } } }, { "prompt": "Translate these frontend review comments into Japanese.", "family": "translate_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "translate_only" ], "exclusive_negative_concepts": [ "translate_only" ], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0.35, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": { "translate_only": [ "translate", "into japanese" ] } } } } ], "near_neighbor": [ { "prompt": "Brainstorm ways to improve this dashboard UX, but do not review code.", "family": "brainstorm_only", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.0, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0.24, "coverage_boost": 0.0, "concept_evidence": { "positive": {}, "negative": { "brainstorm_only": [ "brainstorm" ] } } } }, { "prompt": "Write a frontend review checklist without reviewing the code itself.", "family": "checklist_only", "score": 0.422, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.4, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Fix the failing validation bug in this form component.", "family": "fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ] }, "negative": {} } } } ] } }, "current_holdout_report": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "premerge_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "security_and_a11y": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "brainstorm_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Give this React modal a pre-merge frontend review for a11y, risky rendering, and missing states.", "family": "premerge_review", "score": 0.874, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "state_coverage", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.82, "support_score": 0, "lexical_support": 0.75, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react", "frontend" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } }, { "prompt": "Review this UI diff for accessibility gaps, unsafe form handling, and UX regressions.", "family": "security_and_a11y", "score": 0.496, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.615, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "ui", "form" ], "ui_security": [ "form handling" ] }, "negative": {} } }, "boundary_case": true } ], "should_not_trigger": [ { "prompt": "Create a polished React dashboard page from scratch.", "family": "build_only", "score": 0.057, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Explain keyboard accessibility rules for beginners.", "family": "explain_only", "score": 0.211, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.2, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Discuss possible UI improvements for this checkout flow, but do not inspect the code yet.", "family": "brainstorm_only", "score": 0.053, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "ui" ] }, "negative": {} } } }, { "prompt": "Repair this broken input validation behavior in the form component.", "family": "fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ] }, "negative": {} } } } ] } }, "winner_blind_holdout_report": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_pr_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_premerge": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", "family": "blind_pr_review", "score": 0.504, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.714, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", "family": "blind_premerge", "score": 0.866, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "state_coverage", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.82, "support_score": 0, "lexical_support": 0.647, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "frontend", "form" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a new React settings page and add error handling.", "family": "blind_build_only", "score": 0.061, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Explain why keyboard accessibility matters in forms.", "family": "blind_explain_only", "score": 0.195, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.2, "support_score": 0, "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", "family": "blind_checklist_only", "score": 0.416, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Fix the broken dropdown behavior in this component.", "family": "blind_fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component" ] }, "negative": {} } } } ] } }, "current_blind_holdout_report": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_pr_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_premerge": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", "family": "blind_pr_review", "score": 0.504, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.714, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", "family": "blind_premerge", "score": 0.866, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "state_coverage", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.82, "support_score": 0, "lexical_support": 0.647, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "frontend", "form" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a new React settings page and add error handling.", "family": "blind_build_only", "score": 0.061, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Explain why keyboard accessibility matters in forms.", "family": "blind_explain_only", "score": 0.195, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.2, "support_score": 0, "lexical_support": 0.143, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", "family": "blind_checklist_only", "score": 0.416, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.333, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Fix the broken dropdown behavior in this component.", "family": "blind_fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component" ] }, "negative": {} } } } ] } }, "baseline_blind_holdout_report": { "threshold": 0.48, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_pr_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_premerge": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", "family": "blind_pr_review", "score": 0.492, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.42, "support_score": 0, "lexical_support": 0.571, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", "family": "blind_premerge", "score": 0.857, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "state_coverage", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.82, "support_score": 0, "lexical_support": 0.529, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "frontend", "form" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a new React settings page and add error handling.", "family": "blind_build_only", "score": 0.053, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.2, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } } }, { "prompt": "Explain why keyboard accessibility matters in forms.", "family": "blind_explain_only", "score": 0.207, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.2, "support_score": 0, "lexical_support": 0.286, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", "family": "blind_checklist_only", "score": 0.41, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.25, "negative_penalty": 0, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Fix the broken dropdown behavior in this component.", "family": "blind_fix_only", "score": 0.037, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.0, "negative_penalty": 0, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "component" ] }, "negative": {} } } } ] } }, "winner_judge_blind_holdout_report": { "judge": "rubric-blind-v1", "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_pr_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_premerge": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", "family": "blind_pr_review", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "accessibility", "ui_security", "frontend_scope" ], "strong_positive_concepts": [ "accessibility", "ui_security" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 6, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 4, "negative_vote": 0, "margin": 4, "confidence": 0.82, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } } }, { "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", "family": "blind_premerge", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "review_frontend", "accessibility", "state_coverage", "ux_regression", "frontend_scope" ], "strong_positive_concepts": [ "review_frontend", "accessibility", "state_coverage" ], "trigger_action_hits": [ "run a pre-merge frontend review" ], "input_hits": [], "artifact_hits": [], "capability_overlap": 7, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 6, "negative_vote": 0, "margin": 6, "confidence": 1.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "frontend", "form" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a new React settings page and add error handling.", "family": "blind_build_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "frontend_scope" ], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 1, "negative_vote": 0, "margin": 1, "confidence": 0.58, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Explain why keyboard accessibility matters in forms.", "family": "blind_explain_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "accessibility" ], "strong_positive_concepts": [ "accessibility" ], "trigger_action_hits": [], "input_hits": [ "forms" ], "artifact_hits": [ "forms" ], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "explain" ], "positive_vote": 3, "negative_vote": 1, "margin": 2, "confidence": 0.66, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", "family": "blind_checklist_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "review_frontend", "frontend_scope" ], "strong_positive_concepts": [ "review_frontend" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 3, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 2, "negative_vote": 0, "margin": 2, "confidence": 0.66, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } } }, { "prompt": "Fix the broken dropdown behavior in this component.", "family": "blind_fix_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "frontend_scope" ], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 0, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 0, "negative_vote": 0, "margin": 0, "confidence": 0.5, "concept_evidence": { "positive": { "frontend_scope": [ "component" ] }, "negative": {} } }, "boundary_case": true } ] }, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.703, "rubric_version": "blind-v1" } }, "current_judge_blind_holdout_report": { "judge": "rubric-blind-v1", "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_pr_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_premerge": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", "family": "blind_pr_review", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "accessibility", "ui_security", "frontend_scope" ], "strong_positive_concepts": [ "accessibility", "ui_security" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 6, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 4, "negative_vote": 0, "margin": 4, "confidence": 0.82, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } } }, { "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", "family": "blind_premerge", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "review_frontend", "accessibility", "state_coverage", "ux_regression", "frontend_scope" ], "strong_positive_concepts": [ "review_frontend", "accessibility", "state_coverage" ], "trigger_action_hits": [ "run a pre-merge frontend review" ], "input_hits": [], "artifact_hits": [], "capability_overlap": 7, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 6, "negative_vote": 0, "margin": 6, "confidence": 1.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "frontend", "form" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a new React settings page and add error handling.", "family": "blind_build_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "frontend_scope" ], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 1, "negative_vote": 0, "margin": 1, "confidence": 0.58, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Explain why keyboard accessibility matters in forms.", "family": "blind_explain_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "accessibility" ], "strong_positive_concepts": [ "accessibility" ], "trigger_action_hits": [], "input_hits": [ "forms" ], "artifact_hits": [ "forms" ], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "explain" ], "positive_vote": 3, "negative_vote": 1, "margin": 2, "confidence": 0.66, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", "family": "blind_checklist_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "review_frontend", "frontend_scope" ], "strong_positive_concepts": [ "review_frontend" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 3, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 2, "negative_vote": 0, "margin": 2, "confidence": 0.66, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } } }, { "prompt": "Fix the broken dropdown behavior in this component.", "family": "blind_fix_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "frontend_scope" ], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 0, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 0, "negative_vote": 0, "margin": 0, "confidence": 0.5, "concept_evidence": { "positive": { "frontend_scope": [ "component" ] }, "negative": {} } }, "boundary_case": true } ] }, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.703, "rubric_version": "blind-v1" } }, "baseline_judge_blind_holdout_report": { "judge": "rubric-blind-v1", "judge_explanation": "The blind judge uses a rubric rather than the main threshold scorer. It looks for trigger-action evidence, focused capability evidence, and input or artifact evidence, then blocks on explicit exclusion and exclusive negative signals. This acts as an independent second opinion for blind-holdout prompts.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "blind_pr_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_premerge": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_build_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_explain_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_checklist_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "blind_fix_only": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Review this React pull request for a11y regressions, unsafe rendering, and missing empty states.", "family": "blind_pr_review", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "accessibility", "ui_security", "frontend_scope" ], "strong_positive_concepts": [ "accessibility", "ui_security" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 6, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 4, "negative_vote": 0, "margin": 4, "confidence": 0.82, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "react" ], "ui_security": [ "unsafe rendering" ] }, "negative": {} } } }, { "prompt": "Run a pre-merge frontend review on this diff for form safety, accessibility gaps, and missing states.", "family": "blind_premerge", "predicted_trigger": true, "expected_trigger": true, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "review_frontend", "accessibility", "state_coverage", "ux_regression", "frontend_scope" ], "strong_positive_concepts": [ "review_frontend", "accessibility", "state_coverage" ], "trigger_action_hits": [ "run a pre-merge frontend review" ], "input_hits": [], "artifact_hits": [], "capability_overlap": 7, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 6, "negative_vote": 0, "margin": 6, "confidence": 1.0, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ], "frontend_scope": [ "frontend", "form" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "state_coverage": [ "missing states" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a new React settings page and add error handling.", "family": "blind_build_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "frontend_scope" ], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 1, "negative_vote": 0, "margin": 1, "confidence": 0.58, "concept_evidence": { "positive": { "frontend_scope": [ "react" ] }, "negative": {} } }, "boundary_case": true }, { "prompt": "Explain why keyboard accessibility matters in forms.", "family": "blind_explain_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "accessibility" ], "strong_positive_concepts": [ "accessibility" ], "trigger_action_hits": [], "input_hits": [ "forms" ], "artifact_hits": [ "forms" ], "capability_overlap": 1, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [ "explain" ], "positive_vote": 3, "negative_vote": 1, "margin": 2, "confidence": 0.66, "concept_evidence": { "positive": { "accessibility": [ "accessibility" ] }, "negative": {} } } } ], "near_neighbor": [ { "prompt": "Draft a reusable frontend review checklist, but do not inspect the code.", "family": "blind_checklist_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "review_frontend", "frontend_scope" ], "strong_positive_concepts": [ "review_frontend" ], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 3, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 2, "negative_vote": 0, "margin": 2, "confidence": 0.66, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": {} } } }, { "prompt": "Fix the broken dropdown behavior in this component.", "family": "blind_fix_only", "predicted_trigger": false, "expected_trigger": false, "passed": true, "judge_detail": { "mode": "judge-rubric", "focused_positive_concepts": [ "frontend_scope" ], "strong_positive_concepts": [], "trigger_action_hits": [], "input_hits": [], "artifact_hits": [], "capability_overlap": 0, "exclusive_negative_concepts": [], "nonexclusive_negative_concepts": [], "exclusion_hits": [], "generic_exclusion_hits": [], "positive_vote": 0, "negative_vote": 0, "margin": 0, "confidence": 0.5, "concept_evidence": { "positive": { "frontend_scope": [ "component" ] }, "negative": {} } }, "boundary_case": true } ] }, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.703, "rubric_version": "blind-v1" } }, "winner_adversarial_holdout_report": { "threshold": 0.34, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "adversarial_noisy_premerge_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_quality_gate_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_build_with_a11y_words": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_explain_with_review_terms": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_checklist_not_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_fix_only_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states.", "family": "adversarial_noisy_premerge_review", "score": 0.876, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "ui_security", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.84, "support_score": 0, "lexical_support": 0.542, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "keyboard navigation" ], "frontend_scope": [ "react", "frontend" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "ui_security": [ "unsafe rendering" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } }, { "prompt": "Treat this as a quality gate: review the component for a11y, form handling, and behavior regressions instead of rewriting it.", "family": "adversarial_quality_gate_review", "score": 0.544, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.5, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "component", "form" ], "ui_security": [ "form handling" ], "ux_regression": [ "quality gate" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a React form with accessible labels, safe rendering, and polished empty states.", "family": "adversarial_build_with_a11y_words", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "build_ui" ], "exclusive_negative_concepts": [ "build_ui" ], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.385, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react", "form" ] }, "negative": { "build_ui": [ "build a react" ] } } } }, { "prompt": "Explain keyboard navigation problems in this component and teach me the basics.", "family": "adversarial_explain_with_review_terms", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.083, "negative_penalty": 0.24, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "keyboard navigation" ], "frontend_scope": [ "component" ] }, "negative": { "explain_only": [ "teach me" ] } } } } ], "near_neighbor": [ { "prompt": "Brainstorm a reusable frontend review checklist for later, but do not inspect the code or run a review.", "family": "adversarial_checklist_not_review", "score": 0.035, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.438, "negative_penalty": 0.24, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": { "brainstorm_only": [ "brainstorm" ] } } } }, { "prompt": "Debug this component's form handling bug and fix the unsafe rendering issue.", "family": "adversarial_fix_only_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "fix_only" ], "exclusive_negative_concepts": [ "fix_only" ], "semantic_coverage": 0.22, "support_score": 0, "lexical_support": 0.231, "negative_penalty": 0.18, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ], "ui_security": [ "unsafe rendering", "form handling" ] }, "negative": { "fix_only": [ "debug this component" ] } } } } ] } }, "current_adversarial_holdout_report": { "threshold": 0.34, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "adversarial_noisy_premerge_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_quality_gate_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_build_with_a11y_words": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_explain_with_review_terms": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_checklist_not_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_fix_only_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states.", "family": "adversarial_noisy_premerge_review", "score": 0.876, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "ui_security", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.84, "support_score": 0, "lexical_support": 0.542, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "keyboard navigation" ], "frontend_scope": [ "react", "frontend" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "ui_security": [ "unsafe rendering" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } }, { "prompt": "Treat this as a quality gate: review the component for a11y, form handling, and behavior regressions instead of rewriting it.", "family": "adversarial_quality_gate_review", "score": 0.544, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.5, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "component", "form" ], "ui_security": [ "form handling" ], "ux_regression": [ "quality gate" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a React form with accessible labels, safe rendering, and polished empty states.", "family": "adversarial_build_with_a11y_words", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "build_ui" ], "exclusive_negative_concepts": [ "build_ui" ], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.385, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react", "form" ] }, "negative": { "build_ui": [ "build a react" ] } } } }, { "prompt": "Explain keyboard navigation problems in this component and teach me the basics.", "family": "adversarial_explain_with_review_terms", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.083, "negative_penalty": 0.24, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "keyboard navigation" ], "frontend_scope": [ "component" ] }, "negative": { "explain_only": [ "teach me" ] } } } } ], "near_neighbor": [ { "prompt": "Brainstorm a reusable frontend review checklist for later, but do not inspect the code or run a review.", "family": "adversarial_checklist_not_review", "score": 0.035, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.438, "negative_penalty": 0.24, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": { "brainstorm_only": [ "brainstorm" ] } } } }, { "prompt": "Debug this component's form handling bug and fix the unsafe rendering issue.", "family": "adversarial_fix_only_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "fix_only" ], "exclusive_negative_concepts": [ "fix_only" ], "semantic_coverage": 0.22, "support_score": 0, "lexical_support": 0.231, "negative_penalty": 0.18, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ], "ui_security": [ "unsafe rendering", "form handling" ] }, "negative": { "fix_only": [ "debug this component" ] } } } } ] } }, "baseline_adversarial_holdout_report": { "threshold": 0.34, "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "bucket_stats": { "should_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "should_not_trigger": { "total": 2, "passed": 2, "pass_rate": 1.0 }, "near_neighbor": { "total": 2, "passed": 2, "pass_rate": 1.0 } }, "family_stats": { "adversarial_noisy_premerge_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_quality_gate_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_build_with_a11y_words": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_explain_with_review_terms": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_checklist_not_review": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 }, "adversarial_fix_only_collision": { "total": 1, "passed": 1, "false_positives": 0, "false_negatives": 0, "pass_rate": 1.0 } }, "misfires": [], "results": { "should_trigger": [ { "prompt": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states.", "family": "adversarial_noisy_premerge_review", "score": 0.863, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "review_frontend", "ui_security", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.84, "support_score": 0, "lexical_support": 0.375, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "keyboard navigation" ], "frontend_scope": [ "react", "frontend" ], "review_frontend": [ "frontend review", "pre-merge frontend review" ], "ui_security": [ "unsafe rendering" ], "ux_regression": [ "pre-merge" ] }, "negative": {} } } }, { "prompt": "Treat this as a quality gate: review the component for a11y, form handling, and behavior regressions instead of rewriting it.", "family": "adversarial_quality_gate_review", "score": 0.544, "predicted_trigger": true, "expected_trigger": true, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope", "ui_security", "ux_regression" ], "extra_positive_concepts": [], "matched_negative_concepts": [], "exclusive_negative_concepts": [], "semantic_coverage": 0.5, "support_score": 0, "lexical_support": 0.3, "negative_penalty": 0, "coverage_boost": 0.06, "concept_evidence": { "positive": { "accessibility": [ "a11y" ], "frontend_scope": [ "component", "form" ], "ui_security": [ "form handling" ], "ux_regression": [ "quality gate" ] }, "negative": {} } } } ], "should_not_trigger": [ { "prompt": "Build a React form with accessible labels, safe rendering, and polished empty states.", "family": "adversarial_build_with_a11y_words", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "build_ui" ], "exclusive_negative_concepts": [ "build_ui" ], "semantic_coverage": 0.04, "support_score": 0, "lexical_support": 0.231, "negative_penalty": 0.35, "coverage_boost": 0.0, "concept_evidence": { "positive": { "frontend_scope": [ "react", "form" ] }, "negative": { "build_ui": [ "build a react" ] } } } }, { "prompt": "Explain keyboard navigation problems in this component and teach me the basics.", "family": "adversarial_explain_with_review_terms", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "accessibility", "frontend_scope" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "explain_only" ], "exclusive_negative_concepts": [ "explain_only" ], "semantic_coverage": 0.24, "support_score": 0, "lexical_support": 0.083, "negative_penalty": 0.24, "coverage_boost": 0.04, "concept_evidence": { "positive": { "accessibility": [ "keyboard navigation" ], "frontend_scope": [ "component" ] }, "negative": { "explain_only": [ "teach me" ] } } } } ], "near_neighbor": [ { "prompt": "Brainstorm a reusable frontend review checklist for later, but do not inspect the code or run a review.", "family": "adversarial_checklist_not_review", "score": 0.025, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "review_frontend" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "brainstorm_only" ], "exclusive_negative_concepts": [ "brainstorm_only" ], "semantic_coverage": 0.38, "support_score": 0, "lexical_support": 0.312, "negative_penalty": 0.24, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "frontend" ], "review_frontend": [ "frontend review" ] }, "negative": { "brainstorm_only": [ "brainstorm" ] } } } }, { "prompt": "Debug this component's form handling bug and fix the unsafe rendering issue.", "family": "adversarial_fix_only_collision", "score": 0.0, "predicted_trigger": false, "expected_trigger": false, "passed": true, "score_detail": { "mode": "semantic-intent", "desired_positive_concepts": [ "review_frontend", "accessibility", "ui_security", "state_coverage", "ux_regression", "frontend_scope" ], "matched_desired_concepts": [ "frontend_scope", "ui_security" ], "extra_positive_concepts": [], "matched_negative_concepts": [ "fix_only" ], "exclusive_negative_concepts": [ "fix_only" ], "semantic_coverage": 0.22, "support_score": 0, "lexical_support": 0.077, "negative_penalty": 0.18, "coverage_boost": 0.04, "concept_evidence": { "positive": { "frontend_scope": [ "component", "form" ], "ui_security": [ "unsafe rendering", "form handling" ] }, "negative": { "fix_only": [ "debug this component" ] } } } } ] } }, "candidates": [ { "id": "current", "label": "Current", "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", "strategy": "current", "estimated_tokens": 50, "dev": { "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 3, -1.0, -1.0, -1.0, -0.25, 50 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "guardrail", "label": "Guardrail", "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.", "strategy": "guardrail_template", "estimated_tokens": 62, "dev": { "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 3, -1.0, -1.0, -1.0, -0.25, 62 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "balanced", "label": "Balanced", "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", "strategy": "balanced_template", "estimated_tokens": 64, "dev": { "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 3, -1.0, -1.0, -1.0, -0.25, 64 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "artifact_aware", "label": "Artifact Aware", "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Trigger when requests mention React components, forms, frontend diffs, or UI code and the job is to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", "strategy": "artifact_template", "estimated_tokens": 84, "dev": { "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 3, -1.0, -1.0, -1.0, -0.25, 84 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } }, { "id": "boundary", "label": "Boundary", "description": "Review frontend code for accessibility, UI security, missing states, and UX regressions from React components, forms, frontend diffs, or UI code. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering. Do not use for building a new page from scratch, backend-only review, or explaining frontend concepts.", "strategy": "boundary_template", "estimated_tokens": 90, "dev": { "false_positives": 0, "false_negatives": 3, "precision": 1.0, "recall": 0.25, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "selection_key": [ 0, 3, -1.0, -1.0, -1.0, -0.25, 90 ], "holdout": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 } } ], "selection_logic": { "priority": [ "fewest false positives", "fewest false negatives", "highest near-neighbor pass rate", "highest negative pass rate", "highest precision", "highest recall", "shortest description" ] }, "comparison": { "winner_vs_current_dev": { "baseline_false_positives": 0, "baseline_false_negatives": 3, "improved_false_positives": 0, "improved_false_negatives": 3, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 0.25, "improved_recall": 0.25 }, "winner_vs_current_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_dev": { "baseline_false_positives": 0, "baseline_false_negatives": 3, "improved_false_positives": 0, "improved_false_negatives": 3, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 0.25, "improved_recall": 0.25 }, "winner_vs_baseline_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_judge_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_judge_blind_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_current_adversarial_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 }, "winner_vs_baseline_adversarial_holdout": { "baseline_false_positives": 0, "baseline_false_negatives": 0, "improved_false_positives": 0, "improved_false_negatives": 0, "false_positive_delta": 0, "false_negative_delta": 0, "baseline_precision": 1.0, "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 } }, "acceptance_gates": { "selection_basis": "dev only", "holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.48, "mean_positive_score": 0.685, "mean_non_trigger_score": 0.089, "mean_near_neighbor_score": 0.045, "min_positive_score": 0.496, "max_non_trigger_score": 0.211, "score_gap": 0.285, "threshold_margin": 0.016, "positive_threshold_buffer": 0.016, "negative_threshold_buffer": 0.269, "boundary_case_count": 1, "boundary_case_rate": 0.167, "risk_band": "tight" }, "current_calibration": { "threshold": 0.48, "mean_positive_score": 0.685, "mean_non_trigger_score": 0.089, "mean_near_neighbor_score": 0.045, "min_positive_score": 0.496, "max_non_trigger_score": 0.211, "score_gap": 0.285, "threshold_margin": 0.016, "positive_threshold_buffer": 0.016, "negative_threshold_buffer": 0.269, "boundary_case_count": 1, "boundary_case_rate": 0.167, "risk_band": "tight" }, "baseline_calibration": { "threshold": 0.48, "mean_positive_score": 0.679, "mean_non_trigger_score": 0.087, "mean_near_neighbor_score": 0.045, "min_positive_score": 0.489, "max_non_trigger_score": 0.211, "score_gap": 0.278, "threshold_margin": 0.009, "positive_threshold_buffer": 0.009, "negative_threshold_buffer": 0.269, "boundary_case_count": 1, "boundary_case_rate": 0.167, "risk_band": "tight" }, "winner_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "security_and_a11y", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "security_and_a11y", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "security_and_a11y", "pass_rate": 1.0, "errors": 0 } } }, "blind_holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.48, "mean_positive_score": 0.685, "mean_non_trigger_score": 0.177, "mean_near_neighbor_score": 0.226, "min_positive_score": 0.504, "max_non_trigger_score": 0.416, "score_gap": 0.088, "threshold_margin": 0.024, "positive_threshold_buffer": 0.024, "negative_threshold_buffer": 0.064, "boundary_case_count": 2, "boundary_case_rate": 0.333, "risk_band": "tight" }, "current_calibration": { "threshold": 0.48, "mean_positive_score": 0.685, "mean_non_trigger_score": 0.177, "mean_near_neighbor_score": 0.226, "min_positive_score": 0.504, "max_non_trigger_score": 0.416, "score_gap": 0.088, "threshold_margin": 0.024, "positive_threshold_buffer": 0.024, "negative_threshold_buffer": 0.064, "boundary_case_count": 2, "boundary_case_rate": 0.333, "risk_band": "tight" }, "baseline_calibration": { "threshold": 0.48, "mean_positive_score": 0.674, "mean_non_trigger_score": 0.177, "mean_near_neighbor_score": 0.223, "min_positive_score": 0.492, "max_non_trigger_score": 0.41, "score_gap": 0.082, "threshold_margin": 0.012, "positive_threshold_buffer": 0.012, "negative_threshold_buffer": 0.07, "boundary_case_count": 2, "boundary_case_rate": 0.333, "risk_band": "tight" }, "winner_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_premerge", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_premerge", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_premerge", "pass_rate": 1.0, "errors": 0 } } }, "judge_blind_holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.703, "rubric_version": "blind-v1" } }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.703, "rubric_version": "blind-v1" } }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0, "judge_summary": { "agreement_rate": 1.0, "mean_confidence": 0.703, "rubric_version": "blind-v1" } }, "winner_calibration": null, "current_calibration": null, "baseline_calibration": null, "winner_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_premerge", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_premerge", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "blind_premerge", "pass_rate": 1.0, "errors": 0 } } }, "adversarial_holdout_non_regression": { "winner": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "current": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "baseline": { "false_positives": 0, "false_negatives": 0, "precision": 1.0, "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 }, "winner_calibration": { "threshold": 0.34, "mean_positive_score": 0.71, "mean_non_trigger_score": 0.009, "mean_near_neighbor_score": 0.018, "min_positive_score": 0.544, "max_non_trigger_score": 0.035, "score_gap": 0.509, "threshold_margin": 0.204, "positive_threshold_buffer": 0.204, "negative_threshold_buffer": 0.305, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "current_calibration": { "threshold": 0.34, "mean_positive_score": 0.71, "mean_non_trigger_score": 0.009, "mean_near_neighbor_score": 0.018, "min_positive_score": 0.544, "max_non_trigger_score": 0.035, "score_gap": 0.509, "threshold_margin": 0.204, "positive_threshold_buffer": 0.204, "negative_threshold_buffer": 0.305, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "baseline_calibration": { "threshold": 0.34, "mean_positive_score": 0.704, "mean_non_trigger_score": 0.006, "mean_near_neighbor_score": 0.013, "min_positive_score": 0.544, "max_non_trigger_score": 0.025, "score_gap": 0.519, "threshold_margin": 0.204, "positive_threshold_buffer": 0.204, "negative_threshold_buffer": 0.315, "boundary_case_count": 0, "boundary_case_rate": 0.0, "risk_band": "healthy" }, "winner_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "adversarial_quality_gate_review", "pass_rate": 1.0, "errors": 0 } }, "current_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "adversarial_quality_gate_review", "pass_rate": 1.0, "errors": 0 } }, "baseline_family_health": { "family_count": 6, "clean_family_count": 6, "failing_families": [], "weakest_family": { "family": "adversarial_quality_gate_review", "pass_rate": 1.0, "errors": 0 } } } }, "summary": { "winner_label": "Current", "winner_tokens": 50, "current_tokens": 50, "winner_dev_total_errors": 3, "current_dev_total_errors": 3, "winner_holdout_total_errors": 0, "current_holdout_total_errors": 0, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, "winner_judge_blind_holdout_total_errors": 0, "current_judge_blind_holdout_total_errors": 0, "winner_adversarial_holdout_total_errors": 0, "current_adversarial_holdout_total_errors": 0, "winner_judge_blind_agreement_rate": 1.0, "winner_adversarial_risk_band": "healthy", "winner_adversarial_score_gap": 0.509, "candidate_count": 5, "baseline_tokens": 52, "baseline_dev_total_errors": 3, "baseline_holdout_total_errors": 0, "baseline_blind_holdout_total_errors": 0, "baseline_judge_blind_holdout_total_errors": 0, "baseline_adversarial_holdout_total_errors": 0 } }