diff --git a/README.md b/README.md index 5546a68..ddd2b85 100644 --- a/README.md +++ b/README.md @@ -90,7 +90,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - packaging validation: `openai`, `claude`, and `generic` targets pass contract checks -- description optimization suite: root, team frontend review, and governed incident command all pass the new blind holdout gate; governed incident command still carries one visible holdout miss while blind holdout stays clean +- description optimization suite: root, team frontend review, and governed incident command pass blind and adversarial holdout gates; governed incident command still carries one visible holdout miss, and adversarial calibration plus family drift are now tracked separately - packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected - failure library regressions: anti-pattern families pass automated checks - governance and resource-boundary checks are part of the default test path @@ -161,8 +161,8 @@ Utility scripts that make the meta-skill operational: - `trigger_eval.py`: evaluates trigger descriptions with semantic intent concepts, explicit exclusions, and near-neighbor prompts - `run_eval_suite.py`: runs train/dev/holdout trigger suites, reports family-level regressions, and fails if aggregate regressions appear -- `optimize_description.py`: generates candidate descriptions, scores them on dev, visible holdout, and blind holdout suites, and explains the winning route wording -- `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots +- `optimize_description.py`: generates candidate descriptions, scores them on dev, visible holdout, blind holdout, and adversarial holdout suites, then reports calibration and family health +- `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `context_sizer.py`: estimates context weight and warns when the initial load gets too large - `resource_boundary_check.py`: audits whether detail is split across `SKILL.md`, `references/`, `scripts/`, `assets/`, and `evals/` appropriately @@ -189,7 +189,8 @@ Continuous integration entrypoint that runs the full local regression suite on p - Trigger evaluation now uses a local semantic-intent model with explicit positive concepts, exclusion concepts, and boundary-case reporting. - The sample trigger report now covers a larger positive, negative, and near-neighbor set rather than a tiny demo set. - Train/dev/holdout trigger suites now separate iterative tuning from final verification. -- Description optimization now uses dev for ranking, visible holdout for non-regression, and blind holdout for acceptance without feeding the ranking loop. +- Description optimization now uses dev for ranking, visible holdout for non-regression, blind holdout for acceptance, and adversarial holdout for harder route-collision checks without feeding the ranking loop. +- Description drift history now records adversarial calibration gaps and family coverage, so routing changes can be judged on confidence and family stability rather than raw error counts alone. - Packaging validation now uses explicit contracts and YAML parsing, but it is still a lightweight local validation layer rather than a full platform integration suite. - `evals/failure-cases.md` captures known weak spots that should remain part of regression checks. - `failures/` captures reusable anti-pattern writeups and machine-runnable failure cases for routing, packaging, and authoring failures. diff --git a/evals/README.md b/evals/README.md index e96697c..9be1b14 100644 --- a/evals/README.md +++ b/evals/README.md @@ -7,6 +7,7 @@ Contents: - `trigger_cases.json`: full regression set with `family` labels - `train/`, `dev/`, `holdout/`: split trigger suites for iterative tuning and final verification - `blind_holdout/`: description-optimization acceptance prompts that do not participate in candidate ranking +- `adversarial/`: harder route-collision prompts for description optimization, including noisy positives and deceptive non-trigger requests - `semantic_config.json`: local semantic-intent concepts, exclusions, and weights - `baseline_description.txt`: intentionally weaker trigger description - `improved_description.txt`: current stronger trigger description @@ -22,7 +23,7 @@ Use: python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt python3 scripts/run_eval_suite.py -python3 scripts/run_description_optimization_suite.py --history-snapshot-output evals/history/description_optimization/YYYY-MM-DD-blind-holdout-and-drift-history.json --snapshot-date YYYY-MM-DD +python3 scripts/run_description_optimization_suite.py --history-snapshot-output evals/history/description_optimization/YYYY-MM-DD-adversarial-calibration-and-family-drift.json --snapshot-date YYYY-MM-DD --snapshot-id adversarial-calibration-and-family-drift --snapshot-label "Adversarial Calibration And Family Drift" python3 scripts/render_eval_dashboard.py python3 scripts/render_description_drift_history.py python3 tests/verify_description_optimization.py @@ -44,4 +45,6 @@ Regression scope now includes: - long-context contamination cases where build intent or no-build intent appears after unrelated setup text - family-based reporting across workflow-to-skill, iterate-existing-skill, document-only, one-off, and future-outline cases - holdout verification -- description optimization reports that compare baseline, current, and optimized route wording across dev, holdout, and blind holdout gates +- description optimization reports that compare baseline, current, and optimized route wording across dev, holdout, blind holdout, and adversarial holdout gates +- calibration summaries that surface score gaps, threshold margins, and risk bands for each acceptance gate +- family-level drift history that records which blind and adversarial families stay clean over time diff --git a/evals/adversarial/trigger_cases.json b/evals/adversarial/trigger_cases.json new file mode 100644 index 0000000..ad0d9f8 --- /dev/null +++ b/evals/adversarial/trigger_cases.json @@ -0,0 +1,33 @@ +{ + "recommended_threshold": 0.33, + "should_trigger": [ + { + "family": "adversarial_noisy_operationalize", + "text": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening." + }, + { + "family": "adversarial_multi_asset_boundary", + "text": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary." + } + ], + "should_not_trigger": [ + { + "family": "adversarial_document_collision", + "text": "Create a wiki package from this runbook and do not generate any skill files." + }, + { + "family": "adversarial_summary_translation_collision", + "text": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only." + } + ], + "near_neighbor": [ + { + "family": "adversarial_future_outline_collision", + "text": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note." + }, + { + "family": "adversarial_brainstorm_collision", + "text": "Brainstorm packaging improvements for later and keep this as discussion only." + } + ] +} diff --git a/evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json b/evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json new file mode 100644 index 0000000..f7e2e8e --- /dev/null +++ b/evals/history/description_optimization/2026-04-01-adversarial-calibration-and-family-drift.json @@ -0,0 +1,342 @@ +{ + "snapshot_id": "adversarial-calibration-and-family-drift", + "date": "2026-04-01", + "commit": "local-snapshot", + "label": "Adversarial Calibration And Family Drift", + "targets": [ + { + "name": "yao-meta-skill", + "winner_label": "Current", + "winner_description": "Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.", + "winner_tokens": 65, + "current_tokens": 65, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.33, + "mean_positive_score": 0.672, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "blind_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.026, + "score_gap": 0.366, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.304, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "adversarial_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "family_health": { + "holdout": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy", + "ok": true + }, + { + "name": "team-frontend-review", + "winner_label": "Current", + "winner_description": "Review frontend code for accessibility, UI security, missing states, and UX regressions. Use when asked to review React components, run a pre-merge frontend review, or check a11y and unsafe rendering.", + "winner_tokens": 50, + "current_tokens": 50, + "winner_holdout_fp": 0, + "winner_holdout_fn": 0, + "current_holdout_fp": 0, + "current_holdout_fn": 0, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 0, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 0, + "winner_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.088, + "mean_near_neighbor_score": 0.042, + "min_positive_score": 0.49, + "max_non_trigger_score": 0.211, + "score_gap": 0.279, + "threshold_margin": 0.01, + "positive_threshold_buffer": 0.01, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.226, + "min_positive_score": 0.504, + "max_non_trigger_score": 0.416, + "score_gap": 0.088, + "threshold_margin": 0.024, + "positive_threshold_buffer": 0.024, + "negative_threshold_buffer": 0.064, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.71, + "mean_non_trigger_score": 0.009, + "mean_near_neighbor_score": 0.018, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.035, + "score_gap": 0.509, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.305, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy", + "ok": true + }, + { + "name": "governed-incident-command", + "winner_label": "Current", + "winner_description": "Build governed incident command packets. Use when asked to standardize incident review, run severity assessment, or assemble incident communication.", + "winner_tokens": 37, + "current_tokens": 37, + "winner_holdout_fp": 0, + "winner_holdout_fn": 1, + "current_holdout_fp": 0, + "current_holdout_fn": 1, + "baseline_holdout_fp": 0, + "baseline_holdout_fn": 2, + "winner_blind_holdout_fp": 0, + "winner_blind_holdout_fn": 0, + "current_blind_holdout_fp": 0, + "current_blind_holdout_fn": 0, + "baseline_blind_holdout_fp": 0, + "baseline_blind_holdout_fn": 1, + "winner_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.367, + "mean_non_trigger_score": 0.019, + "mean_near_neighbor_score": 0.038, + "min_positive_score": 0.08, + "max_non_trigger_score": 0.058, + "score_gap": 0.022, + "threshold_margin": -0.4, + "positive_threshold_buffer": -0.4, + "negative_threshold_buffer": 0.422, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "overlap" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.899, + "mean_non_trigger_score": 0.168, + "mean_near_neighbor_score": 0.138, + "min_positive_score": 0.798, + "max_non_trigger_score": 0.386, + "score_gap": 0.412, + "threshold_margin": 0.094, + "positive_threshold_buffer": 0.318, + "negative_threshold_buffer": 0.094, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "healthy" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.827, + "mean_non_trigger_score": 0.027, + "mean_near_neighbor_score": 0.011, + "min_positive_score": 0.654, + "max_non_trigger_score": 0.056, + "score_gap": 0.598, + "threshold_margin": 0.284, + "positive_threshold_buffer": 0.314, + "negative_threshold_buffer": 0.284, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy", + "ok": true + } + ], + "notes": [ + "recorded family-level blind and adversarial routing evidence", + "published calibration and drift history for description optimization" + ] +} diff --git a/evals/history/description_optimization/README.md b/evals/history/description_optimization/README.md index 92ad808..da74bad 100644 --- a/evals/history/description_optimization/README.md +++ b/evals/history/description_optimization/README.md @@ -9,6 +9,9 @@ Each snapshot records: - per-target winner token counts - visible holdout errors - blind holdout errors when available +- adversarial holdout errors when available +- calibration summaries for blind and adversarial gates when available +- family coverage summaries for blind and adversarial gates when available - short drift notes for route wording changes or new acceptance gates Use `python3 scripts/render_description_drift_history.py` to rebuild `reports/description_drift_history.md`. diff --git a/examples/README.md b/examples/README.md index 40d8915..a7a6e2f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -19,6 +19,7 @@ Some examples also include: - `optimization/`: example-specific baseline descriptions, semantic configs, dev/holdout suites, and route-optimization reports - `optimization/blind_holdout/`: acceptance prompts used only after the winner is chosen on dev +- `optimization/adversarial/`: harder route-collision prompts used to measure noisy positives, deceptive negatives, and family-level calibration The complex example is intentionally thicker than the others and includes: diff --git a/examples/governed-incident-command/optimization/adversarial/trigger_cases.json b/examples/governed-incident-command/optimization/adversarial/trigger_cases.json new file mode 100644 index 0000000..0b79d02 --- /dev/null +++ b/examples/governed-incident-command/optimization/adversarial/trigger_cases.json @@ -0,0 +1,33 @@ +{ + "recommended_threshold": 0.34, + "should_trigger": [ + { + "family": "adversarial_governed_packet_from_noise", + "text": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions." + }, + { + "family": "adversarial_operational_skill_route", + "text": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications." + } + ], + "should_not_trigger": [ + { + "family": "adversarial_single_update_collision", + "text": "Write one short customer outage update from these incident notes and stop there." + }, + { + "family": "adversarial_explain_and_summary_collision", + "text": "Explain incident severity and summarize the outage timeline for training." + } + ], + "near_neighbor": [ + { + "family": "adversarial_future_playbook_not_packet", + "text": "Sketch a future incident review playbook for later, but do not package a reusable workflow." + }, + { + "family": "adversarial_debugging_collision", + "text": "Troubleshoot the outage and fix the service before we talk about reusable incident packets." + } + ] +} diff --git a/examples/governed-incident-command/optimization/reports/description_optimization.json b/examples/governed-incident-command/optimization/reports/description_optimization.json index 014f851..93bf03a 100644 --- a/examples/governed-incident-command/optimization/reports/description_optimization.json +++ b/examples/governed-incident-command/optimization/reports/description_optimization.json @@ -3300,6 +3300,1048 @@ ] } }, + "winner_adversarial_holdout_report": { + "threshold": 0.34, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_governed_packet_from_noise": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_operational_skill_route": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_single_update_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_explain_and_summary_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_playbook_not_packet": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_debugging_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.", + "family": "adversarial_governed_packet_from_noise", + "score": 1.0, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [ + "comms_and_owners", + "governed_workflow", + "severity_assessment" + ], + "extra_positive_concepts": [ + "incident_inputs", + "incident_packet" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 1.0, + "support_score": 0.52, + "lexical_support": 0.263, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "owner" + ], + "governed_workflow": [ + "governed" + ], + "incident_inputs": [ + "alerts", + "transcripts", + "stakeholder requirements" + ], + "incident_packet": [ + "governed incident packet" + ], + "severity_assessment": [ + "severity assessment", + "severity" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.", + "family": "adversarial_operational_skill_route", + "score": 0.654, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [ + "governed_workflow", + "severity_assessment" + ], + "extra_positive_concepts": [ + "reusable_packaging" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.636, + "support_score": 0.04, + "lexical_support": 0.235, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "governed_workflow": [ + "maintained", + "operational skill" + ], + "reusable_packaging": [ + "skill", + "package" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Write one short customer outage update from these incident notes and stop there.", + "family": "adversarial_single_update_collision", + "score": 0.056, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "incident_inputs" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0.2, + "lexical_support": 0.077, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "incident_inputs": [ + "incident notes" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Explain incident severity and summarize the outage timeline for training.", + "family": "adversarial_explain_and_summary_collision", + "score": 0.032, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [ + "severity_assessment" + ], + "extra_positive_concepts": [ + "incident_inputs" + ], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.409, + "support_score": 0.2, + "lexical_support": 0.2, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "incident_inputs": [ + "timeline" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": { + "explain_only": [ + "explain incident severity" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.", + "family": "adversarial_future_playbook_not_packet", + "score": 0.021, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "reusable_packaging" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0.04, + "lexical_support": 0.143, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reusable_packaging": [ + "reusable", + "workflow", + "package" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.", + "family": "adversarial_debugging_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "reusable_packaging" + ], + "matched_negative_concepts": [ + "generic_debugging" + ], + "exclusive_negative_concepts": [ + "generic_debugging" + ], + "semantic_coverage": 0.0, + "support_score": 0.04, + "lexical_support": 0.154, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reusable_packaging": [ + "reusable" + ] + }, + "negative": { + "generic_debugging": [ + "fix the service" + ] + } + } + } + } + ] + } + }, + "current_adversarial_holdout_report": { + "threshold": 0.34, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_governed_packet_from_noise": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_operational_skill_route": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_single_update_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_explain_and_summary_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_playbook_not_packet": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_debugging_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.", + "family": "adversarial_governed_packet_from_noise", + "score": 1.0, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [ + "comms_and_owners", + "governed_workflow", + "severity_assessment" + ], + "extra_positive_concepts": [ + "incident_inputs", + "incident_packet" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 1.0, + "support_score": 0.52, + "lexical_support": 0.263, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "owner" + ], + "governed_workflow": [ + "governed" + ], + "incident_inputs": [ + "alerts", + "transcripts", + "stakeholder requirements" + ], + "incident_packet": [ + "governed incident packet" + ], + "severity_assessment": [ + "severity assessment", + "severity" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.", + "family": "adversarial_operational_skill_route", + "score": 0.654, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [ + "governed_workflow", + "severity_assessment" + ], + "extra_positive_concepts": [ + "reusable_packaging" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.636, + "support_score": 0.04, + "lexical_support": 0.235, + "negative_penalty": 0, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "governed_workflow": [ + "maintained", + "operational skill" + ], + "reusable_packaging": [ + "skill", + "package" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Write one short customer outage update from these incident notes and stop there.", + "family": "adversarial_single_update_collision", + "score": 0.056, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "incident_inputs" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0.2, + "lexical_support": 0.077, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "incident_inputs": [ + "incident notes" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Explain incident severity and summarize the outage timeline for training.", + "family": "adversarial_explain_and_summary_collision", + "score": 0.032, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [ + "severity_assessment" + ], + "extra_positive_concepts": [ + "incident_inputs" + ], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.409, + "support_score": 0.2, + "lexical_support": 0.2, + "negative_penalty": 0.26, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "incident_inputs": [ + "timeline" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": { + "explain_only": [ + "explain incident severity" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.", + "family": "adversarial_future_playbook_not_packet", + "score": 0.021, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "reusable_packaging" + ], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.0, + "support_score": 0.04, + "lexical_support": 0.143, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reusable_packaging": [ + "reusable", + "workflow", + "package" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.", + "family": "adversarial_debugging_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "severity_assessment", + "comms_and_owners", + "governed_workflow" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [ + "reusable_packaging" + ], + "matched_negative_concepts": [ + "generic_debugging" + ], + "exclusive_negative_concepts": [ + "generic_debugging" + ], + "semantic_coverage": 0.0, + "support_score": 0.04, + "lexical_support": 0.154, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reusable_packaging": [ + "reusable" + ] + }, + "negative": { + "generic_debugging": [ + "fix the service" + ] + } + } + } + } + ] + } + }, + "baseline_adversarial_holdout_report": { + "threshold": 0.34, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_governed_packet_from_noise": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_operational_skill_route": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_single_update_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_explain_and_summary_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_playbook_not_packet": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_debugging_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the incident chatter and assemble a governed incident packet from alerts, transcripts, and stakeholder requirements, including severity assessment and owner actions.", + "family": "adversarial_governed_packet_from_noise", + "score": 0.99, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "incident_packet", + "incident_inputs", + "severity_assessment", + "comms_and_owners", + "governed_workflow", + "reusable_packaging" + ], + "matched_desired_concepts": [ + "comms_and_owners", + "governed_workflow", + "incident_inputs", + "incident_packet", + "severity_assessment" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.96, + "support_score": 0, + "lexical_support": 0.579, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "comms_and_owners": [ + "owner" + ], + "governed_workflow": [ + "governed" + ], + "incident_inputs": [ + "alerts", + "transcripts", + "stakeholder requirements" + ], + "incident_packet": [ + "governed incident packet" + ], + "severity_assessment": [ + "severity assessment", + "severity" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Treat this outage as a maintained operational skill: standardize the incident review, assess severity, and package communications.", + "family": "adversarial_operational_skill_route", + "score": 0.387, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "incident_packet", + "incident_inputs", + "severity_assessment", + "comms_and_owners", + "governed_workflow", + "reusable_packaging" + ], + "matched_desired_concepts": [ + "governed_workflow", + "reusable_packaging", + "severity_assessment" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.32, + "support_score": 0, + "lexical_support": 0.412, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "governed_workflow": [ + "maintained", + "operational skill" + ], + "reusable_packaging": [ + "skill", + "package" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": {} + } + }, + "boundary_case": true + } + ], + "should_not_trigger": [ + { + "prompt": "Write one short customer outage update from these incident notes and stop there.", + "family": "adversarial_single_update_collision", + "score": 0.215, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "incident_packet", + "incident_inputs", + "severity_assessment", + "comms_and_owners", + "governed_workflow", + "reusable_packaging" + ], + "matched_desired_concepts": [ + "incident_inputs" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.2, + "support_score": 0, + "lexical_support": 0.385, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "incident_inputs": [ + "incident notes" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Explain incident severity and summarize the outage timeline for training.", + "family": "adversarial_explain_and_summary_collision", + "score": 0.012, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "incident_packet", + "incident_inputs", + "severity_assessment", + "comms_and_owners", + "governed_workflow", + "reusable_packaging" + ], + "matched_desired_concepts": [ + "incident_inputs", + "severity_assessment" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.38, + "support_score": 0, + "lexical_support": 0.4, + "negative_penalty": 0.26, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "incident_inputs": [ + "timeline" + ], + "severity_assessment": [ + "severity" + ] + }, + "negative": { + "explain_only": [ + "explain incident severity" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Sketch a future incident review playbook for later, but do not package a reusable workflow.", + "family": "adversarial_future_playbook_not_packet", + "score": 0.065, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "incident_packet", + "incident_inputs", + "severity_assessment", + "comms_and_owners", + "governed_workflow", + "reusable_packaging" + ], + "matched_desired_concepts": [ + "reusable_packaging" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.04, + "support_score": 0, + "lexical_support": 0.357, + "negative_penalty": 0, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reusable_packaging": [ + "reusable", + "workflow", + "package" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Troubleshoot the outage and fix the service before we talk about reusable incident packets.", + "family": "adversarial_debugging_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "incident_packet", + "incident_inputs", + "severity_assessment", + "comms_and_owners", + "governed_workflow", + "reusable_packaging" + ], + "matched_desired_concepts": [ + "reusable_packaging" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "generic_debugging" + ], + "exclusive_negative_concepts": [ + "generic_debugging" + ], + "semantic_coverage": 0.04, + "support_score": 0, + "lexical_support": 0.308, + "negative_penalty": 0.3, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reusable_packaging": [ + "reusable" + ] + }, + "negative": { + "generic_debugging": [ + "fix the service" + ] + } + } + } + } + ] + } + }, "candidates": [ { "id": "current", @@ -3545,6 +4587,30 @@ "improved_precision": 1.0, "baseline_recall": 0.5, "improved_recall": 1.0 + }, + "winner_vs_current_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 } }, "acceptance_gates": { @@ -3573,6 +4639,104 @@ "recall": 0.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.367, + "mean_non_trigger_score": 0.019, + "mean_near_neighbor_score": 0.038, + "min_positive_score": 0.08, + "max_non_trigger_score": 0.058, + "score_gap": 0.022, + "threshold_margin": -0.4, + "positive_threshold_buffer": -0.4, + "negative_threshold_buffer": 0.422, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "overlap" + }, + "current_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.367, + "mean_non_trigger_score": 0.019, + "mean_near_neighbor_score": 0.038, + "min_positive_score": 0.08, + "max_non_trigger_score": 0.058, + "score_gap": 0.022, + "threshold_margin": -0.4, + "positive_threshold_buffer": -0.4, + "negative_threshold_buffer": 0.422, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "overlap" + }, + "baseline_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.358, + "mean_non_trigger_score": 0.058, + "mean_near_neighbor_score": 0.117, + "min_positive_score": 0.321, + "max_non_trigger_score": 0.216, + "score_gap": 0.105, + "threshold_margin": -0.159, + "positive_threshold_buffer": -0.159, + "negative_threshold_buffer": 0.264, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "overlap" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 4, + "failing_families": [ + { + "family": "governed_workflow", + "errors": 1, + "pass_rate": 0.0 + }, + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } } }, "blind_holdout_non_regression": { @@ -3599,6 +4763,188 @@ "recall": 0.5, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.899, + "mean_non_trigger_score": 0.168, + "mean_near_neighbor_score": 0.138, + "min_positive_score": 0.798, + "max_non_trigger_score": 0.386, + "score_gap": 0.412, + "threshold_margin": 0.094, + "positive_threshold_buffer": 0.318, + "negative_threshold_buffer": 0.094, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "healthy" + }, + "current_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.899, + "mean_non_trigger_score": 0.168, + "mean_near_neighbor_score": 0.138, + "min_positive_score": 0.798, + "max_non_trigger_score": 0.386, + "score_gap": 0.412, + "threshold_margin": 0.094, + "positive_threshold_buffer": 0.318, + "negative_threshold_buffer": 0.094, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "healthy" + }, + "baseline_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.622, + "mean_non_trigger_score": 0.149, + "mean_near_neighbor_score": 0.194, + "min_positive_score": 0.452, + "max_non_trigger_score": 0.387, + "score_gap": 0.065, + "threshold_margin": -0.028, + "positive_threshold_buffer": -0.028, + "negative_threshold_buffer": 0.093, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "overlap" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "blind_governed_workflow", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "blind_governed_workflow", + "pass_rate": 0.0, + "errors": 1 + } + } + }, + "adversarial_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.827, + "mean_non_trigger_score": 0.027, + "mean_near_neighbor_score": 0.011, + "min_positive_score": 0.654, + "max_non_trigger_score": 0.056, + "score_gap": 0.598, + "threshold_margin": 0.284, + "positive_threshold_buffer": 0.314, + "negative_threshold_buffer": 0.284, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "current_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.827, + "mean_non_trigger_score": 0.027, + "mean_near_neighbor_score": 0.011, + "min_positive_score": 0.654, + "max_non_trigger_score": 0.056, + "score_gap": 0.598, + "threshold_margin": 0.284, + "positive_threshold_buffer": 0.314, + "negative_threshold_buffer": 0.284, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "baseline_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.689, + "mean_non_trigger_score": 0.073, + "mean_near_neighbor_score": 0.033, + "min_positive_score": 0.387, + "max_non_trigger_score": 0.215, + "score_gap": 0.172, + "threshold_margin": 0.047, + "positive_threshold_buffer": 0.047, + "negative_threshold_buffer": 0.125, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "watch" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } } } }, @@ -3612,10 +4958,15 @@ "current_holdout_total_errors": 1, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_total_errors": 0, + "current_adversarial_holdout_total_errors": 0, + "winner_adversarial_risk_band": "healthy", + "winner_adversarial_score_gap": 0.598, "candidate_count": 5, "baseline_tokens": 93, "baseline_dev_total_errors": 1, "baseline_holdout_total_errors": 2, - "baseline_blind_holdout_total_errors": 1 + "baseline_blind_holdout_total_errors": 1, + "baseline_adversarial_holdout_total_errors": 0 } } diff --git a/examples/governed-incident-command/optimization/reports/description_optimization.md b/examples/governed-incident-command/optimization/reports/description_optimization.md index 6a8e415..6e98b8b 100644 --- a/examples/governed-incident-command/optimization/reports/description_optimization.md +++ b/examples/governed-incident-command/optimization/reports/description_optimization.md @@ -26,6 +26,23 @@ Build governed incident command packets. Use when asked to standardize incident | --- | ---: | ---: | ---: | ---: | ---: | ---: | | Holdout | 0 | 1 | 0 | 1 | 0 | 2 | | Blind Holdout | 0 | 0 | 0 | 0 | 0 | 1 | +| Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 | + +## Calibration + +| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap | +| --- | ---: | --- | ---: | ---: | ---: | +| Holdout | 0.022 | overlap | 0.0 | 0.022 | 0.105 | +| Blind Holdout | 0.412 | healthy | 0.167 | 0.412 | 0.065 | +| Adversarial Holdout | 0.598 | healthy | 0.0 | 0.598 | 0.172 | + +## Family Health + +| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families | +| --- | --- | --- | --- | --- | +| Holdout | 5/6 | packet_assembly (1 errors) | 5/6 | 4/6 | +| Blind Holdout | 6/6 | blind_summary_only (0 errors) | 6/6 | 5/6 | +| Adversarial Holdout | 6/6 | adversarial_single_update_collision (0 errors) | 6/6 | 6/6 | ## Selection Logic diff --git a/examples/team-frontend-review/optimization/adversarial/trigger_cases.json b/examples/team-frontend-review/optimization/adversarial/trigger_cases.json new file mode 100644 index 0000000..2cb4e63 --- /dev/null +++ b/examples/team-frontend-review/optimization/adversarial/trigger_cases.json @@ -0,0 +1,33 @@ +{ + "recommended_threshold": 0.34, + "should_trigger": [ + { + "family": "adversarial_noisy_premerge_review", + "text": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states." + }, + { + "family": "adversarial_quality_gate_review", + "text": "Treat this as a quality gate: review the component for a11y, form handling, and behavior regressions instead of rewriting it." + } + ], + "should_not_trigger": [ + { + "family": "adversarial_build_with_a11y_words", + "text": "Build a React form with accessible labels, safe rendering, and polished empty states." + }, + { + "family": "adversarial_explain_with_review_terms", + "text": "Explain keyboard navigation problems in this component and teach me the basics." + } + ], + "near_neighbor": [ + { + "family": "adversarial_checklist_not_review", + "text": "Brainstorm a reusable frontend review checklist for later, but do not inspect the code or run a review." + }, + { + "family": "adversarial_fix_only_collision", + "text": "Debug this component's form handling bug and fix the unsafe rendering issue." + } + ] +} diff --git a/examples/team-frontend-review/optimization/reports/description_optimization.json b/examples/team-frontend-review/optimization/reports/description_optimization.json index 64dcc0c..28c36da 100644 --- a/examples/team-frontend-review/optimization/reports/description_optimization.json +++ b/examples/team-frontend-review/optimization/reports/description_optimization.json @@ -3942,6 +3942,1158 @@ ] } }, + "winner_adversarial_holdout_report": { + "threshold": 0.34, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_premerge_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_quality_gate_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_build_with_a11y_words": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_explain_with_review_terms": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_checklist_not_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_fix_only_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states.", + "family": "adversarial_noisy_premerge_review", + "score": 0.876, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "review_frontend", + "ui_security", + "ux_regression" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.84, + "support_score": 0, + "lexical_support": 0.545, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "accessibility": [ + "keyboard navigation" + ], + "frontend_scope": [ + "react", + "frontend" + ], + "review_frontend": [ + "frontend review", + "pre-merge frontend review" + ], + "ui_security": [ + "unsafe rendering" + ], + "ux_regression": [ + "pre-merge" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Treat this as a quality gate: review the component for a11y, form handling, and behavior regressions instead of rewriting it.", + "family": "adversarial_quality_gate_review", + "score": 0.544, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "ui_security", + "ux_regression" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.5, + "support_score": 0, + "lexical_support": 0.3, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "accessibility": [ + "a11y" + ], + "frontend_scope": [ + "component", + "form" + ], + "ui_security": [ + "form handling" + ], + "ux_regression": [ + "quality gate" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Build a React form with accessible labels, safe rendering, and polished empty states.", + "family": "adversarial_build_with_a11y_words", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "build_ui" + ], + "exclusive_negative_concepts": [ + "build_ui" + ], + "semantic_coverage": 0.04, + "support_score": 0, + "lexical_support": 0.385, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "react", + "form" + ] + }, + "negative": { + "build_ui": [ + "build a react" + ] + } + } + } + }, + { + "prompt": "Explain keyboard navigation problems in this component and teach me the basics.", + "family": "adversarial_explain_with_review_terms", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.24, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "accessibility": [ + "keyboard navigation" + ], + "frontend_scope": [ + "component" + ] + }, + "negative": { + "explain_only": [ + "teach me" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Brainstorm a reusable frontend review checklist for later, but do not inspect the code or run a review.", + "family": "adversarial_checklist_not_review", + "score": 0.035, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope", + "review_frontend" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.38, + "support_score": 0, + "lexical_support": 0.438, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "frontend" + ], + "review_frontend": [ + "frontend review" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ] + } + } + } + }, + { + "prompt": "Debug this component's form handling bug and fix the unsafe rendering issue.", + "family": "adversarial_fix_only_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope", + "ui_security" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "fix_only" + ], + "exclusive_negative_concepts": [ + "fix_only" + ], + "semantic_coverage": 0.22, + "support_score": 0, + "lexical_support": 0.231, + "negative_penalty": 0.18, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "component", + "form" + ], + "ui_security": [ + "unsafe rendering", + "form handling" + ] + }, + "negative": { + "fix_only": [ + "debug this component" + ] + } + } + } + } + ] + } + }, + "current_adversarial_holdout_report": { + "threshold": 0.34, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_premerge_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_quality_gate_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_build_with_a11y_words": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_explain_with_review_terms": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_checklist_not_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_fix_only_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states.", + "family": "adversarial_noisy_premerge_review", + "score": 0.876, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "review_frontend", + "ui_security", + "ux_regression" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.84, + "support_score": 0, + "lexical_support": 0.545, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "accessibility": [ + "keyboard navigation" + ], + "frontend_scope": [ + "react", + "frontend" + ], + "review_frontend": [ + "frontend review", + "pre-merge frontend review" + ], + "ui_security": [ + "unsafe rendering" + ], + "ux_regression": [ + "pre-merge" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Treat this as a quality gate: review the component for a11y, form handling, and behavior regressions instead of rewriting it.", + "family": "adversarial_quality_gate_review", + "score": 0.544, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "ui_security", + "ux_regression" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.5, + "support_score": 0, + "lexical_support": 0.3, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "accessibility": [ + "a11y" + ], + "frontend_scope": [ + "component", + "form" + ], + "ui_security": [ + "form handling" + ], + "ux_regression": [ + "quality gate" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Build a React form with accessible labels, safe rendering, and polished empty states.", + "family": "adversarial_build_with_a11y_words", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "build_ui" + ], + "exclusive_negative_concepts": [ + "build_ui" + ], + "semantic_coverage": 0.04, + "support_score": 0, + "lexical_support": 0.385, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "react", + "form" + ] + }, + "negative": { + "build_ui": [ + "build a react" + ] + } + } + } + }, + { + "prompt": "Explain keyboard navigation problems in this component and teach me the basics.", + "family": "adversarial_explain_with_review_terms", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.24, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "accessibility": [ + "keyboard navigation" + ], + "frontend_scope": [ + "component" + ] + }, + "negative": { + "explain_only": [ + "teach me" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Brainstorm a reusable frontend review checklist for later, but do not inspect the code or run a review.", + "family": "adversarial_checklist_not_review", + "score": 0.035, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope", + "review_frontend" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.38, + "support_score": 0, + "lexical_support": 0.438, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "frontend" + ], + "review_frontend": [ + "frontend review" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ] + } + } + } + }, + { + "prompt": "Debug this component's form handling bug and fix the unsafe rendering issue.", + "family": "adversarial_fix_only_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope", + "ui_security" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "fix_only" + ], + "exclusive_negative_concepts": [ + "fix_only" + ], + "semantic_coverage": 0.22, + "support_score": 0, + "lexical_support": 0.231, + "negative_penalty": 0.18, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "component", + "form" + ], + "ui_security": [ + "unsafe rendering", + "form handling" + ] + }, + "negative": { + "fix_only": [ + "debug this component" + ] + } + } + } + } + ] + } + }, + "baseline_adversarial_holdout_report": { + "threshold": 0.34, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_premerge_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_quality_gate_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_build_with_a11y_words": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_explain_with_review_terms": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_checklist_not_review": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_fix_only_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the feature-request chatter and run a pre-merge frontend review on this React diff for keyboard navigation, unsafe rendering, and missing empty states.", + "family": "adversarial_noisy_premerge_review", + "score": 0.862, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "review_frontend", + "ui_security", + "ux_regression" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.84, + "support_score": 0, + "lexical_support": 0.364, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "accessibility": [ + "keyboard navigation" + ], + "frontend_scope": [ + "react", + "frontend" + ], + "review_frontend": [ + "frontend review", + "pre-merge frontend review" + ], + "ui_security": [ + "unsafe rendering" + ], + "ux_regression": [ + "pre-merge" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Treat this as a quality gate: review the component for a11y, form handling, and behavior regressions instead of rewriting it.", + "family": "adversarial_quality_gate_review", + "score": 0.544, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope", + "ui_security", + "ux_regression" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.5, + "support_score": 0, + "lexical_support": 0.3, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "accessibility": [ + "a11y" + ], + "frontend_scope": [ + "component", + "form" + ], + "ui_security": [ + "form handling" + ], + "ux_regression": [ + "quality gate" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Build a React form with accessible labels, safe rendering, and polished empty states.", + "family": "adversarial_build_with_a11y_words", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "build_ui" + ], + "exclusive_negative_concepts": [ + "build_ui" + ], + "semantic_coverage": 0.04, + "support_score": 0, + "lexical_support": 0.231, + "negative_penalty": 0.35, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "react", + "form" + ] + }, + "negative": { + "build_ui": [ + "build a react" + ] + } + } + } + }, + { + "prompt": "Explain keyboard navigation problems in this component and teach me the basics.", + "family": "adversarial_explain_with_review_terms", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "accessibility", + "frontend_scope" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "explain_only" + ], + "exclusive_negative_concepts": [ + "explain_only" + ], + "semantic_coverage": 0.24, + "support_score": 0, + "lexical_support": 0.083, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "accessibility": [ + "keyboard navigation" + ], + "frontend_scope": [ + "component" + ] + }, + "negative": { + "explain_only": [ + "teach me" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Brainstorm a reusable frontend review checklist for later, but do not inspect the code or run a review.", + "family": "adversarial_checklist_not_review", + "score": 0.025, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope", + "review_frontend" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only" + ], + "exclusive_negative_concepts": [ + "brainstorm_only" + ], + "semantic_coverage": 0.38, + "support_score": 0, + "lexical_support": 0.312, + "negative_penalty": 0.24, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "frontend" + ], + "review_frontend": [ + "frontend review" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ] + } + } + } + }, + { + "prompt": "Debug this component's form handling bug and fix the unsafe rendering issue.", + "family": "adversarial_fix_only_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "review_frontend", + "accessibility", + "ui_security", + "state_coverage", + "ux_regression", + "frontend_scope" + ], + "matched_desired_concepts": [ + "frontend_scope", + "ui_security" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "fix_only" + ], + "exclusive_negative_concepts": [ + "fix_only" + ], + "semantic_coverage": 0.22, + "support_score": 0, + "lexical_support": 0.077, + "negative_penalty": 0.18, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "frontend_scope": [ + "component", + "form" + ], + "ui_security": [ + "unsafe rendering", + "form handling" + ] + }, + "negative": { + "fix_only": [ + "debug this component" + ] + } + } + } + } + ] + } + }, "candidates": [ { "id": "current", @@ -4187,6 +5339,30 @@ "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 + }, + "winner_vs_current_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 } }, "acceptance_gates": { @@ -4215,6 +5391,81 @@ "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.088, + "mean_near_neighbor_score": 0.042, + "min_positive_score": 0.49, + "max_non_trigger_score": 0.211, + "score_gap": 0.279, + "threshold_margin": 0.01, + "positive_threshold_buffer": 0.01, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "current_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.088, + "mean_near_neighbor_score": 0.042, + "min_positive_score": 0.49, + "max_non_trigger_score": 0.211, + "score_gap": 0.279, + "threshold_margin": 0.01, + "positive_threshold_buffer": 0.01, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "baseline_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.675, + "mean_non_trigger_score": 0.086, + "mean_near_neighbor_score": 0.042, + "min_positive_score": 0.483, + "max_non_trigger_score": 0.211, + "score_gap": 0.272, + "threshold_margin": 0.003, + "positive_threshold_buffer": 0.003, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } } }, "blind_holdout_non_regression": { @@ -4241,6 +5492,182 @@ "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.226, + "min_positive_score": 0.504, + "max_non_trigger_score": 0.416, + "score_gap": 0.088, + "threshold_margin": 0.024, + "positive_threshold_buffer": 0.024, + "negative_threshold_buffer": 0.064, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "current_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.226, + "min_positive_score": 0.504, + "max_non_trigger_score": 0.416, + "score_gap": 0.088, + "threshold_margin": 0.024, + "positive_threshold_buffer": 0.024, + "negative_threshold_buffer": 0.064, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "baseline_calibration": { + "threshold": 0.48, + "mean_positive_score": 0.673, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.223, + "min_positive_score": 0.492, + "max_non_trigger_score": 0.41, + "score_gap": 0.082, + "threshold_margin": 0.012, + "positive_threshold_buffer": 0.012, + "negative_threshold_buffer": 0.07, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "adversarial_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.71, + "mean_non_trigger_score": 0.009, + "mean_near_neighbor_score": 0.018, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.035, + "score_gap": 0.509, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.305, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "current_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.71, + "mean_non_trigger_score": 0.009, + "mean_near_neighbor_score": 0.018, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.035, + "score_gap": 0.509, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.305, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "baseline_calibration": { + "threshold": 0.34, + "mean_positive_score": 0.703, + "mean_non_trigger_score": 0.006, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.025, + "score_gap": 0.519, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.315, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } } } }, @@ -4254,10 +5681,15 @@ "current_holdout_total_errors": 0, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_total_errors": 0, + "current_adversarial_holdout_total_errors": 0, + "winner_adversarial_risk_band": "healthy", + "winner_adversarial_score_gap": 0.509, "candidate_count": 5, "baseline_tokens": 52, "baseline_dev_total_errors": 4, "baseline_holdout_total_errors": 0, - "baseline_blind_holdout_total_errors": 0 + "baseline_blind_holdout_total_errors": 0, + "baseline_adversarial_holdout_total_errors": 0 } } diff --git a/examples/team-frontend-review/optimization/reports/description_optimization.md b/examples/team-frontend-review/optimization/reports/description_optimization.md index 0a3647e..73d4d2a 100644 --- a/examples/team-frontend-review/optimization/reports/description_optimization.md +++ b/examples/team-frontend-review/optimization/reports/description_optimization.md @@ -26,6 +26,23 @@ Review frontend code for accessibility, UI security, missing states, and UX regr | --- | ---: | ---: | ---: | ---: | ---: | ---: | | Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | +| Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 | + +## Calibration + +| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap | +| --- | ---: | --- | ---: | ---: | ---: | +| Holdout | 0.279 | tight | 0.167 | 0.279 | 0.272 | +| Blind Holdout | 0.088 | tight | 0.333 | 0.088 | 0.082 | +| Adversarial Holdout | 0.509 | healthy | 0.0 | 0.509 | 0.519 | + +## Family Health + +| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families | +| --- | --- | --- | --- | --- | +| Holdout | 6/6 | security_and_a11y (0 errors) | 6/6 | 6/6 | +| Blind Holdout | 6/6 | blind_premerge (0 errors) | 6/6 | 6/6 | +| Adversarial Holdout | 6/6 | adversarial_quality_gate_review (0 errors) | 6/6 | 6/6 | ## Selection Logic diff --git a/reports/context_budget.json b/reports/context_budget.json index 7c529fc..ab2ca04 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 898, - "other_text_tokens": 170683, + "other_text_tokens": 193933, "estimated_initial_load_tokens": 989, - "estimated_total_text_tokens": 171581, - "relevant_file_count": 101, + "estimated_total_text_tokens": 194831, + "relevant_file_count": 103, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.4 diff --git a/reports/description_drift_history.md b/reports/description_drift_history.md index 9984a06..8e1aa01 100644 --- a/reports/description_drift_history.md +++ b/reports/description_drift_history.md @@ -1,10 +1,27 @@ # Description Drift History -| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Drift Note | -| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- | -| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | `Current` | 65 | 0 | 0 | - | - | initial description optimization suite without blind holdout | -| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | `Current` | 50 | 0 | 0 | - | - | compressed example description and stored first holdout report | -| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | `Current` | 37 | 0 | 1 | - | - | compressed governed description and reduced visible holdout misses | -| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | tokens stable; blind gate added with 0 errors; holdout stable at 0 | -| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | tokens stable; blind gate added with 0 errors; holdout stable at 0 | -| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | tokens stable; blind gate added with 0 errors; holdout stable at 1 | +| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Drift Note | +| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | +| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | `Current` | 65 | 0 | 0 | - | - | - | - | - | - | initial description optimization suite without blind holdout | +| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | `Current` | 50 | 0 | 0 | - | - | - | - | - | - | compressed example description and stored first holdout report | +| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | `Current` | 37 | 0 | 1 | - | - | - | - | - | - | compressed governed description and reduced visible holdout misses | +| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 0 | 0.834 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.834; risk n/a -> healthy | +| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0.509 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 0; adversarial calibration +0.509; risk n/a -> healthy | +| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 0 | 0.598 | healthy | tokens stable; blind stable at 0; adversarial gate added with 0 errors; holdout stable at 1; adversarial calibration +0.598; risk n/a -> healthy | +| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | +| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 0 | +| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | - | - | - | - | tokens stable; blind gate added with 0 errors; holdout stable at 1 | + +## Family Coverage + +| Date | Label | Target | Blind Families | Adversarial Families | +| --- | --- | --- | --- | --- | +| 2026-03-31 | Description Optimization Suite | `yao-meta-skill` | - | - | +| 2026-03-31 | Description Optimization Suite | `team-frontend-review` | - | - | +| 2026-03-31 | Description Optimization Suite | `governed-incident-command` | - | - | +| 2026-04-01 | Adversarial Calibration And Family Drift | `yao-meta-skill` | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=adversarial_summary_translation_collision | +| 2026-04-01 | Adversarial Calibration And Family Drift | `team-frontend-review` | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=adversarial_quality_gate_review | +| 2026-04-01 | Adversarial Calibration And Family Drift | `governed-incident-command` | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=adversarial_single_update_collision | +| 2026-04-01 | Blind Holdout And Drift History | `yao-meta-skill` | - | - | +| 2026-04-01 | Blind Holdout And Drift History | `team-frontend-review` | - | - | +| 2026-04-01 | Blind Holdout And Drift History | `governed-incident-command` | - | - | diff --git a/reports/description_optimization.json b/reports/description_optimization.json index b68f6b7..ffde081 100644 --- a/reports/description_optimization.json +++ b/reports/description_optimization.json @@ -7449,6 +7449,1215 @@ ] } }, + "winner_adversarial_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_multi_asset_boundary": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_document_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_summary_translation_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_outline_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_brainstorm_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", + "family": "adversarial_noisy_operationalize", + "score": 0.834, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.263, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "hardening" + ], + "reuse_package": [ + "package", + "maintained", + "operationalize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", + "family": "adversarial_multi_asset_boundary", + "score": 0.842, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.368, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this routine" + ], + "eval_optimize": [ + "stress test", + "route boundary" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "prompt history" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Create a wiki package from this runbook and do not generate any skill files.", + "family": "adversarial_document_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.429, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "document_only": [ + "wiki" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", + "family": "adversarial_summary_translation_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive", + "summarize_only", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only", + "translate_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 1.25, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "document_only": [ + "knowledge base" + ], + "no_build_directive": [ + "keep it as documentation only" + ], + "summarize_only": [ + "summarize" + ], + "translate_only": [ + "translate", + "into japanese" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", + "family": "adversarial_future_outline_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill" + ], + "no_build_directive": [ + "stop short of making a full skill", + "plain note", + "keep it as a plain note" + ] + } + } + } + }, + { + "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", + "family": "adversarial_brainstorm_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.182, + "negative_penalty": 0.88, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ], + "future_outline": [ + "for later" + ], + "no_build_directive": [ + "discussion only" + ] + } + } + } + } + ] + } + }, + "current_adversarial_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_multi_asset_boundary": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_document_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_summary_translation_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_outline_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_brainstorm_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", + "family": "adversarial_noisy_operationalize", + "score": 0.834, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.263, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "hardening" + ], + "reuse_package": [ + "package", + "maintained", + "operationalize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", + "family": "adversarial_multi_asset_boundary", + "score": 0.842, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.818, + "support_score": 0, + "lexical_support": 0.368, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this routine" + ], + "eval_optimize": [ + "stress test", + "route boundary" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "prompt history" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Create a wiki package from this runbook and do not generate any skill files.", + "family": "adversarial_document_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.364, + "support_score": 0.06, + "lexical_support": 0.429, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "document_only": [ + "wiki" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", + "family": "adversarial_summary_translation_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive", + "summarize_only", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only", + "translate_only" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.2, + "negative_penalty": 1.25, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "document_only": [ + "knowledge base" + ], + "no_build_directive": [ + "keep it as documentation only" + ], + "summarize_only": [ + "summarize" + ], + "translate_only": [ + "translate", + "into japanese" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", + "family": "adversarial_future_outline_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.25, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill" + ], + "no_build_directive": [ + "stop short of making a full skill", + "plain note", + "keep it as a plain note" + ] + } + } + } + }, + { + "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", + "family": "adversarial_brainstorm_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill", + "team_operationalize" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.182, + "support_score": 0, + "lexical_support": 0.182, + "negative_penalty": 0.88, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ], + "future_outline": [ + "for later" + ], + "no_build_directive": [ + "discussion only" + ] + } + } + } + } + ] + } + }, + "baseline_adversarial_holdout_report": { + "threshold": 0.33, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores are driven primarily by semantic intent coverage: packaging intent, workflow-to-skill transformation intent, reuse/distribution intent, and eval intent. Explicit exclusions such as summary-only, translation-only, one-off, document-only, or do-not-build directives apply direct penalties and can override otherwise similar wording.", + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 2, + "passed": 2, + "pass_rate": 1.0 + } + }, + "family_stats": { + "adversarial_noisy_operationalize": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_multi_asset_boundary": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_document_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_summary_translation_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_future_outline_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + }, + "adversarial_brainstorm_collision": { + "total": 1, + "passed": 1, + "false_positives": 0, + "false_negatives": 0, + "pass_rate": 1.0 + } + }, + "misfires": [], + "results": { + "should_trigger": [ + { + "prompt": "Ignore the noisy meeting chatter and operationalize this support escalation workflow into a maintained skill package with trigger hardening.", + "family": "adversarial_noisy_operationalize", + "score": 0.845, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0, + "lexical_support": 0.053, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "skill package" + ], + "eval_optimize": [ + "hardening" + ], + "reuse_package": [ + "package", + "maintained", + "operationalize" + ], + "transform_workflow": [ + "workflow" + ] + }, + "negative": {} + } + } + }, + { + "prompt": "Codify this routine into a reusable capability from docs, transcripts, and prompt history, then stress test the route boundary.", + "family": "adversarial_multi_asset_boundary", + "score": 0.845, + "predicted_trigger": true, + "expected_trigger": true, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "build_skill", + "eval_optimize", + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [], + "exclusive_negative_concepts": [], + "semantic_coverage": 0.849, + "support_score": 0, + "lexical_support": 0.053, + "negative_penalty": 0, + "coverage_boost": 0.06, + "concept_evidence": { + "positive": { + "build_skill": [ + "codify this routine" + ], + "eval_optimize": [ + "stress test", + "route boundary" + ], + "reuse_package": [ + "reusable" + ], + "transform_workflow": [ + "prompt history" + ] + }, + "negative": {} + } + } + } + ], + "should_not_trigger": [ + { + "prompt": "Create a wiki package from this runbook and do not generate any skill files.", + "family": "adversarial_document_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package", + "transform_workflow" + ], + "extra_positive_concepts": [ + "multi_asset" + ], + "matched_negative_concepts": [ + "document_only", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "no_build_directive" + ], + "semantic_coverage": 0.377, + "support_score": 0.06, + "lexical_support": 0.143, + "negative_penalty": 0.6, + "coverage_boost": 0.04, + "concept_evidence": { + "positive": { + "multi_asset": [ + "files" + ], + "reuse_package": [ + "package" + ], + "transform_workflow": [ + "runbook" + ] + }, + "negative": { + "document_only": [ + "wiki" + ], + "no_build_directive": [ + "do not generate any skill files" + ] + } + } + } + }, + { + "prompt": "Summarize this SOP, translate it into Japanese for the knowledge base, and keep it as documentation only.", + "family": "adversarial_summary_translation_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "transform_workflow" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "document_only", + "no_build_directive", + "summarize_only", + "translate_only" + ], + "exclusive_negative_concepts": [ + "no_build_directive", + "summarize_only", + "translate_only" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.067, + "negative_penalty": 1.25, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "transform_workflow": [ + "sop" + ] + }, + "negative": { + "document_only": [ + "knowledge base" + ], + "no_build_directive": [ + "keep it as documentation only" + ], + "summarize_only": [ + "summarize" + ], + "translate_only": [ + "translate", + "into japanese" + ] + } + } + } + } + ], + "near_neighbor": [ + { + "prompt": "Outline a possible future skill from these notes, but stop short of making a full skill and keep it as a plain note.", + "family": "adversarial_future_outline_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.0, + "support_score": 0, + "lexical_support": 0.05, + "negative_penalty": 0.64, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": {}, + "negative": { + "future_outline": [ + "future skill", + "possible future skill" + ], + "no_build_directive": [ + "stop short of making a full skill", + "plain note", + "keep it as a plain note" + ] + } + } + } + }, + { + "prompt": "Brainstorm packaging improvements for later and keep this as discussion only.", + "family": "adversarial_brainstorm_collision", + "score": 0.0, + "predicted_trigger": false, + "expected_trigger": false, + "passed": true, + "score_detail": { + "mode": "semantic-intent", + "desired_positive_concepts": [ + "build_skill", + "transform_workflow", + "reuse_package", + "eval_optimize", + "iterate_existing_skill" + ], + "matched_desired_concepts": [ + "reuse_package" + ], + "extra_positive_concepts": [], + "matched_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "exclusive_negative_concepts": [ + "brainstorm_only", + "future_outline", + "no_build_directive" + ], + "semantic_coverage": 0.189, + "support_score": 0, + "lexical_support": 0.091, + "negative_penalty": 0.88, + "coverage_boost": 0.0, + "concept_evidence": { + "positive": { + "reuse_package": [ + "packaging" + ] + }, + "negative": { + "brainstorm_only": [ + "brainstorm" + ], + "future_outline": [ + "for later" + ], + "no_build_directive": [ + "discussion only" + ] + } + } + } + } + ] + } + }, "candidates": [ { "id": "current", @@ -7726,6 +8935,30 @@ "improved_precision": 1.0, "baseline_recall": 1.0, "improved_recall": 1.0 + }, + "winner_vs_current_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 + }, + "winner_vs_baseline_adversarial_holdout": { + "baseline_false_positives": 0, + "baseline_false_negatives": 0, + "improved_false_positives": 0, + "improved_false_negatives": 0, + "false_positive_delta": 0, + "false_negative_delta": 0, + "baseline_precision": 1.0, + "improved_precision": 1.0, + "baseline_recall": 1.0, + "improved_recall": 1.0 } }, "acceptance_gates": { @@ -7754,6 +8987,81 @@ "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.672, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "current_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.672, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "baseline_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.648, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.416, + "max_non_trigger_score": 0.0, + "score_gap": 0.416, + "threshold_margin": 0.086, + "positive_threshold_buffer": 0.086, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "winner_family_health": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } } }, "blind_holdout_non_regression": { @@ -7780,6 +9088,182 @@ "recall": 1.0, "near_neighbor_pass_rate": 1.0, "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.026, + "score_gap": 0.366, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.304, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "current_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.026, + "score_gap": 0.366, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.304, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "baseline_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.393, + "mean_non_trigger_score": 0.01, + "mean_near_neighbor_score": 0.012, + "min_positive_score": 0.393, + "max_non_trigger_score": 0.024, + "score_gap": 0.369, + "threshold_margin": 0.063, + "positive_threshold_buffer": 0.063, + "negative_threshold_buffer": 0.306, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "adversarial_holdout_non_regression": { + "winner": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "current": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "baseline": { + "false_positives": 0, + "false_negatives": 0, + "precision": 1.0, + "recall": 1.0, + "near_neighbor_pass_rate": 1.0, + "should_not_trigger_pass_rate": 1.0 + }, + "winner_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "current_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "baseline_calibration": { + "threshold": 0.33, + "mean_positive_score": 0.845, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.845, + "max_non_trigger_score": 0.0, + "score_gap": 0.845, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.515, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "winner_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + }, + "current_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + }, + "baseline_family_health": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } } } }, @@ -7793,10 +9277,15 @@ "current_holdout_total_errors": 0, "winner_blind_holdout_total_errors": 0, "current_blind_holdout_total_errors": 0, + "winner_adversarial_holdout_total_errors": 0, + "current_adversarial_holdout_total_errors": 0, + "winner_adversarial_risk_band": "healthy", + "winner_adversarial_score_gap": 0.834, "candidate_count": 6, "baseline_tokens": 8, "baseline_dev_total_errors": 2, "baseline_holdout_total_errors": 0, - "baseline_blind_holdout_total_errors": 0 + "baseline_blind_holdout_total_errors": 0, + "baseline_adversarial_holdout_total_errors": 0 } } diff --git a/reports/description_optimization.md b/reports/description_optimization.md index 86ad308..911410f 100644 --- a/reports/description_optimization.md +++ b/reports/description_optimization.md @@ -27,6 +27,23 @@ Create, refactor, evaluate, and package agent skills from workflows, prompts, tr | --- | ---: | ---: | ---: | ---: | ---: | ---: | | Holdout | 0 | 0 | 0 | 0 | 0 | 0 | | Blind Holdout | 0 | 0 | 0 | 0 | 0 | 0 | +| Adversarial Holdout | 0 | 0 | 0 | 0 | 0 | 0 | + +## Calibration + +| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap | +| --- | ---: | --- | ---: | ---: | ---: | +| Holdout | 0.468 | healthy | 0.0 | 0.468 | 0.416 | +| Blind Holdout | 0.366 | watch | 0.333 | 0.366 | 0.369 | +| Adversarial Holdout | 0.834 | healthy | 0.0 | 0.834 | 0.845 | + +## Family Health + +| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families | +| --- | --- | --- | --- | --- | +| Holdout | 12/12 | workflow_to_skill (0 errors) | 12/12 | 12/12 | +| Blind Holdout | 6/6 | blind_operationalize (0 errors) | 6/6 | 6/6 | +| Adversarial Holdout | 6/6 | adversarial_summary_translation_collision (0 errors) | 6/6 | 6/6 | ## Selection Logic diff --git a/reports/description_optimization_suite.json b/reports/description_optimization_suite.json index 05e1235..effcbae 100644 --- a/reports/description_optimization_suite.json +++ b/reports/description_optimization_suite.json @@ -19,7 +19,93 @@ "baseline_blind_holdout_fp": 0, "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, - "drift_note": "blind holdout gate active", + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.33, + "mean_positive_score": 0.672, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.468, + "max_non_trigger_score": 0.0, + "score_gap": 0.468, + "threshold_margin": 0.138, + "positive_threshold_buffer": 0.138, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + }, + "blind_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.399, + "mean_non_trigger_score": 0.013, + "mean_near_neighbor_score": 0.013, + "min_positive_score": 0.392, + "max_non_trigger_score": 0.026, + "score_gap": 0.366, + "threshold_margin": 0.062, + "positive_threshold_buffer": 0.062, + "negative_threshold_buffer": 0.304, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "watch" + }, + "adversarial_holdout": { + "threshold": 0.33, + "mean_positive_score": 0.838, + "mean_non_trigger_score": 0.0, + "mean_near_neighbor_score": 0.0, + "min_positive_score": 0.834, + "max_non_trigger_score": 0.0, + "score_gap": 0.834, + "threshold_margin": 0.33, + "positive_threshold_buffer": 0.504, + "negative_threshold_buffer": 0.33, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "family_health": { + "holdout": { + "family_count": 12, + "clean_family_count": 12, + "failing_families": [], + "weakest_family": { + "family": "workflow_to_skill", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_operationalize", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_summary_translation_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, adversarial, and calibration gates active", "ok": true }, { @@ -41,7 +127,93 @@ "baseline_blind_holdout_fp": 0, "baseline_blind_holdout_fn": 0, "winner_blind_holdout_total_errors": 0, - "drift_note": "blind holdout gate active", + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.088, + "mean_near_neighbor_score": 0.042, + "min_positive_score": 0.49, + "max_non_trigger_score": 0.211, + "score_gap": 0.279, + "threshold_margin": 0.01, + "positive_threshold_buffer": 0.01, + "negative_threshold_buffer": 0.269, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "tight" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.681, + "mean_non_trigger_score": 0.177, + "mean_near_neighbor_score": 0.226, + "min_positive_score": 0.504, + "max_non_trigger_score": 0.416, + "score_gap": 0.088, + "threshold_margin": 0.024, + "positive_threshold_buffer": 0.024, + "negative_threshold_buffer": 0.064, + "boundary_case_count": 2, + "boundary_case_rate": 0.333, + "risk_band": "tight" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.71, + "mean_non_trigger_score": 0.009, + "mean_near_neighbor_score": 0.018, + "min_positive_score": 0.544, + "max_non_trigger_score": 0.035, + "score_gap": 0.509, + "threshold_margin": 0.204, + "positive_threshold_buffer": 0.204, + "negative_threshold_buffer": 0.305, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "security_and_a11y", + "pass_rate": 1.0, + "errors": 0 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_premerge", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_quality_gate_review", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, adversarial, and calibration gates active", "ok": true }, { @@ -63,7 +235,99 @@ "baseline_blind_holdout_fp": 0, "baseline_blind_holdout_fn": 1, "winner_blind_holdout_total_errors": 0, - "drift_note": "blind holdout gate active", + "winner_adversarial_holdout_fp": 0, + "winner_adversarial_holdout_fn": 0, + "current_adversarial_holdout_fp": 0, + "current_adversarial_holdout_fn": 0, + "baseline_adversarial_holdout_fp": 0, + "baseline_adversarial_holdout_fn": 0, + "winner_adversarial_holdout_total_errors": 0, + "calibration": { + "holdout": { + "threshold": 0.48, + "mean_positive_score": 0.367, + "mean_non_trigger_score": 0.019, + "mean_near_neighbor_score": 0.038, + "min_positive_score": 0.08, + "max_non_trigger_score": 0.058, + "score_gap": 0.022, + "threshold_margin": -0.4, + "positive_threshold_buffer": -0.4, + "negative_threshold_buffer": 0.422, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "overlap" + }, + "blind_holdout": { + "threshold": 0.48, + "mean_positive_score": 0.899, + "mean_non_trigger_score": 0.168, + "mean_near_neighbor_score": 0.138, + "min_positive_score": 0.798, + "max_non_trigger_score": 0.386, + "score_gap": 0.412, + "threshold_margin": 0.094, + "positive_threshold_buffer": 0.318, + "negative_threshold_buffer": 0.094, + "boundary_case_count": 1, + "boundary_case_rate": 0.167, + "risk_band": "healthy" + }, + "adversarial_holdout": { + "threshold": 0.34, + "mean_positive_score": 0.827, + "mean_non_trigger_score": 0.027, + "mean_near_neighbor_score": 0.011, + "min_positive_score": 0.654, + "max_non_trigger_score": 0.056, + "score_gap": 0.598, + "threshold_margin": 0.284, + "positive_threshold_buffer": 0.314, + "negative_threshold_buffer": 0.284, + "boundary_case_count": 0, + "boundary_case_rate": 0.0, + "risk_band": "healthy" + } + }, + "family_health": { + "holdout": { + "family_count": 6, + "clean_family_count": 5, + "failing_families": [ + { + "family": "packet_assembly", + "errors": 1, + "pass_rate": 0.0 + } + ], + "weakest_family": { + "family": "packet_assembly", + "pass_rate": 0.0, + "errors": 1 + } + }, + "blind_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "blind_summary_only", + "pass_rate": 1.0, + "errors": 0 + } + }, + "adversarial_holdout": { + "family_count": 6, + "clean_family_count": 6, + "failing_families": [], + "weakest_family": { + "family": "adversarial_single_update_collision", + "pass_rate": 1.0, + "errors": 0 + } + } + }, + "drift_note": "blind, adversarial, and calibration gates active", "ok": true } ], diff --git a/reports/description_optimization_suite.md b/reports/description_optimization_suite.md index 8fad23c..53fdf31 100644 --- a/reports/description_optimization_suite.md +++ b/reports/description_optimization_suite.md @@ -1,7 +1,15 @@ # Description Optimization Suite -| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Current Blind FN | Baseline Blind FN | Status | -| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | -| `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 0 | ok | -| `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 0 | ok | -| `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 1 | ok | +| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Status | +| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- | +| `yao-meta-skill` | `Current` | 65 | 0 | 0 | 0 | 0 | 0 | 0 | 0.834 | healthy | ok | +| `team-frontend-review` | `Current` | 50 | 0 | 0 | 0 | 0 | 0 | 0 | 0.509 | healthy | ok | +| `governed-incident-command` | `Current` | 37 | 0 | 1 | 0 | 0 | 0 | 0 | 0.598 | healthy | ok | + +## Family Coverage + +| Target | Blind Families | Adversarial Families | +| --- | --- | --- | +| `yao-meta-skill` | 6/6 clean; weakest=blind_operationalize | 6/6 clean; weakest=adversarial_summary_translation_collision | +| `team-frontend-review` | 6/6 clean; weakest=blind_premerge | 6/6 clean; weakest=adversarial_quality_gate_review | +| `governed-incident-command` | 6/6 clean; weakest=blind_summary_only | 6/6 clean; weakest=adversarial_single_update_collision | diff --git a/scripts/optimize_description.py b/scripts/optimize_description.py index c3896c5..4958ca1 100644 --- a/scripts/optimize_description.py +++ b/scripts/optimize_description.py @@ -184,6 +184,122 @@ def error_tuple(report: dict | None) -> tuple[int, int] | None: return (report["false_positives"], report["false_negatives"]) +def safe_round(value: float | None) -> float | None: + if value is None: + return None + return round(value, 3) + + +def summarize_family_health(report: dict | None) -> dict | None: + if not report: + return None + + family_stats = report.get("family_stats", {}) + ordered = sorted( + family_stats.items(), + key=lambda item: ( + item[1].get("false_positives", 0) + item[1].get("false_negatives", 0), + item[1].get("pass_rate") or 0, + -(item[1].get("total") or 0), + item[0], + ), + ) + weakest = ordered[-1] if ordered else None + failing = [] + for family, stats in family_stats.items(): + errors = stats.get("false_positives", 0) + stats.get("false_negatives", 0) + if errors: + failing.append({"family": family, "errors": errors, "pass_rate": stats.get("pass_rate")}) + failing.sort(key=lambda item: (-item["errors"], item["pass_rate"] or 0, item["family"])) + clean_count = sum( + 1 + for stats in family_stats.values() + if (stats.get("false_positives", 0) + stats.get("false_negatives", 0)) == 0 + ) + return { + "family_count": len(family_stats), + "clean_family_count": clean_count, + "failing_families": failing, + "weakest_family": { + "family": weakest[0], + "pass_rate": weakest[1].get("pass_rate"), + "errors": weakest[1].get("false_positives", 0) + weakest[1].get("false_negatives", 0), + } + if weakest + else None, + } + + +def summarize_calibration(report: dict | None, threshold: float | None) -> dict | None: + if not report or threshold is None: + return None + + positive_scores = [item["score"] for item in report["results"].get("should_trigger", [])] + should_not_scores = [item["score"] for item in report["results"].get("should_not_trigger", [])] + near_scores = [item["score"] for item in report["results"].get("near_neighbor", [])] + non_trigger_scores = should_not_scores + near_scores + total_cases = sum(len(items) for items in report["results"].values()) + boundary_cases = sum( + 1 + for items in report["results"].values() + for item in items + if item.get("boundary_case") + ) + + min_positive = min(positive_scores) if positive_scores else None + max_non_trigger = max(non_trigger_scores) if non_trigger_scores else None + positive_threshold_buffer = (min_positive - threshold) if min_positive is not None else None + negative_threshold_buffer = (threshold - max_non_trigger) if max_non_trigger is not None else None + score_gap = (min_positive - max_non_trigger) if min_positive is not None and max_non_trigger is not None else None + margin_candidates = [value for value in (positive_threshold_buffer, negative_threshold_buffer) if value is not None] + threshold_margin = min(margin_candidates) if margin_candidates else None + + risk_band = "healthy" + if report["false_positives"] or report["false_negatives"] or (score_gap is not None and score_gap < 0): + risk_band = "overlap" + elif threshold_margin is not None and threshold_margin < 0.03: + risk_band = "tight" + elif threshold_margin is not None and threshold_margin < 0.08: + risk_band = "watch" + elif total_cases and (boundary_cases / total_cases) > 0.25: + risk_band = "watch" + + return { + "threshold": safe_round(threshold), + "mean_positive_score": safe_round(sum(positive_scores) / len(positive_scores)) if positive_scores else None, + "mean_non_trigger_score": safe_round(sum(non_trigger_scores) / len(non_trigger_scores)) if non_trigger_scores else None, + "mean_near_neighbor_score": safe_round(sum(near_scores) / len(near_scores)) if near_scores else None, + "min_positive_score": safe_round(min_positive), + "max_non_trigger_score": safe_round(max_non_trigger), + "score_gap": safe_round(score_gap), + "threshold_margin": safe_round(threshold_margin), + "positive_threshold_buffer": safe_round(positive_threshold_buffer), + "negative_threshold_buffer": safe_round(negative_threshold_buffer), + "boundary_case_count": boundary_cases, + "boundary_case_rate": safe_round(boundary_cases / total_cases) if total_cases else None, + "risk_band": risk_band, + } + + +def build_gate_summary( + winner_report: dict | None, + current_report: dict | None, + baseline_report: dict | None, + threshold: float | None, +) -> dict: + return { + "winner": summarize_gate_report(winner_report), + "current": summarize_gate_report(current_report), + "baseline": summarize_gate_report(baseline_report), + "winner_calibration": summarize_calibration(winner_report, threshold), + "current_calibration": summarize_calibration(current_report, threshold), + "baseline_calibration": summarize_calibration(baseline_report, threshold), + "winner_family_health": summarize_family_health(winner_report), + "current_family_health": summarize_family_health(current_report), + "baseline_family_health": summarize_family_health(baseline_report), + } + + def optimize( current_description: str, dev_cases: dict, @@ -191,12 +307,18 @@ def optimize( config: dict, baseline_description: str | None = None, blind_holdout_cases: dict | None = None, + adversarial_cases: dict | None = None, ) -> dict: dev_threshold = dev_cases.get("recommended_threshold", 0.48) holdout_threshold = holdout_cases.get("recommended_threshold", dev_threshold) if holdout_cases else dev_threshold blind_holdout_threshold = ( blind_holdout_cases.get("recommended_threshold", holdout_threshold) if blind_holdout_cases else holdout_threshold ) + adversarial_threshold = ( + adversarial_cases.get("recommended_threshold", blind_holdout_threshold) + if adversarial_cases + else blind_holdout_threshold + ) candidates = [] for candidate in build_candidates(current_description, config): @@ -234,6 +356,19 @@ def optimize( baseline["description"], blind_holdout_cases, blind_holdout_threshold, config ) + adversarial_reports = {} + if adversarial_cases: + adversarial_reports["current"] = evaluate( + current["candidate"]["description"], adversarial_cases, adversarial_threshold, config + ) + adversarial_reports["winner"] = evaluate( + winner["candidate"]["description"], adversarial_cases, adversarial_threshold, config + ) + if baseline: + adversarial_reports["baseline"] = evaluate( + baseline["description"], adversarial_cases, adversarial_threshold, config + ) + report = { "current_description": sentence(current_description), "current_candidate": current["candidate"], @@ -246,6 +381,9 @@ def optimize( "winner_blind_holdout_report": blind_reports.get("winner"), "current_blind_holdout_report": blind_reports.get("current"), "baseline_blind_holdout_report": blind_reports.get("baseline"), + "winner_adversarial_holdout_report": adversarial_reports.get("winner"), + "current_adversarial_holdout_report": adversarial_reports.get("current"), + "baseline_adversarial_holdout_report": adversarial_reports.get("baseline"), "candidates": [item["candidate"] for item in candidates], "selection_logic": { "priority": [ @@ -273,19 +411,37 @@ def optimize( "winner_vs_baseline_blind_holdout": compare_reports(blind_reports["baseline"], blind_reports["winner"]) if blind_reports.get("baseline") and blind_reports.get("winner") else None, + "winner_vs_current_adversarial_holdout": compare_reports( + adversarial_reports["current"], adversarial_reports["winner"] + ) + if adversarial_reports.get("current") and adversarial_reports.get("winner") + else None, + "winner_vs_baseline_adversarial_holdout": compare_reports( + adversarial_reports["baseline"], adversarial_reports["winner"] + ) + if adversarial_reports.get("baseline") and adversarial_reports.get("winner") + else None, }, "acceptance_gates": { "selection_basis": "dev only", - "holdout_non_regression": { - "winner": summarize_gate_report(winner["holdout_report"]), - "current": summarize_gate_report(current["holdout_report"]), - "baseline": summarize_gate_report(baseline["holdout"]) if baseline else None, - }, - "blind_holdout_non_regression": { - "winner": summarize_gate_report(blind_reports.get("winner")), - "current": summarize_gate_report(blind_reports.get("current")), - "baseline": summarize_gate_report(blind_reports.get("baseline")), - }, + "holdout_non_regression": build_gate_summary( + winner["holdout_report"], + current["holdout_report"], + baseline["holdout"] if baseline else None, + holdout_threshold if holdout_cases else None, + ), + "blind_holdout_non_regression": build_gate_summary( + blind_reports.get("winner"), + blind_reports.get("current"), + blind_reports.get("baseline"), + blind_holdout_threshold if blind_holdout_cases else None, + ), + "adversarial_holdout_non_regression": build_gate_summary( + adversarial_reports.get("winner"), + adversarial_reports.get("current"), + adversarial_reports.get("baseline"), + adversarial_threshold if adversarial_cases else None, + ), }, } report["summary"] = { @@ -308,6 +464,18 @@ def optimize( "current_blind_holdout_total_errors": sum(error_tuple(blind_reports.get("current"))) if blind_reports.get("current") else None, + "winner_adversarial_holdout_total_errors": sum(error_tuple(adversarial_reports.get("winner"))) + if adversarial_reports.get("winner") + else None, + "current_adversarial_holdout_total_errors": sum(error_tuple(adversarial_reports.get("current"))) + if adversarial_reports.get("current") + else None, + "winner_adversarial_risk_band": report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"]["risk_band"] + if report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"] + else None, + "winner_adversarial_score_gap": report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"]["score_gap"] + if report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"] + else None, "candidate_count": len(report["candidates"]), } if baseline: @@ -321,6 +489,9 @@ def optimize( report["summary"]["baseline_blind_holdout_total_errors"] = ( sum(error_tuple(blind_reports.get("baseline"))) if blind_reports.get("baseline") else None ) + report["summary"]["baseline_adversarial_holdout_total_errors"] = ( + sum(error_tuple(adversarial_reports.get("baseline"))) if adversarial_reports.get("baseline") else None + ) return report @@ -366,6 +537,7 @@ def render_markdown(report: dict, title: str) -> str: for gate_name, gate in ( ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), ): winner_gate = gate.get("winner") or {} current_gate = gate.get("current") or {} @@ -376,6 +548,58 @@ def render_markdown(report: dict, title: str) -> str: f"| {gate_name} | {winner_gate.get('false_positives', '-')} | {winner_gate.get('false_negatives', '-')} | {current_gate.get('false_positives', '-')} | {current_gate.get('false_negatives', '-')} | {baseline_gate.get('false_positives', '-')} | {baseline_gate.get('false_negatives', '-')} |" ) + lines.extend( + [ + "", + "## Calibration", + "", + "| Gate | Winner Gap | Winner Risk | Winner Boundary Rate | Current Gap | Baseline Gap |", + "| --- | ---: | --- | ---: | ---: | ---: |", + ] + ) + for gate_name, gate in ( + ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), + ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), + ): + winner_calibration = gate.get("winner_calibration") or {} + current_calibration = gate.get("current_calibration") or {} + baseline_calibration = gate.get("baseline_calibration") or {} + if not winner_calibration and not current_calibration and not baseline_calibration: + continue + lines.append( + f"| {gate_name} | {winner_calibration.get('score_gap', '-')} | {winner_calibration.get('risk_band', '-')} | {winner_calibration.get('boundary_case_rate', '-')} | {current_calibration.get('score_gap', '-')} | {baseline_calibration.get('score_gap', '-')} |" + ) + + lines.extend( + [ + "", + "## Family Health", + "", + "| Gate | Winner Clean Families | Winner Weakest Family | Current Clean Families | Baseline Clean Families |", + "| --- | --- | --- | --- | --- |", + ] + ) + for gate_name, gate in ( + ("Holdout", report["acceptance_gates"]["holdout_non_regression"]), + ("Blind Holdout", report["acceptance_gates"]["blind_holdout_non_regression"]), + ("Adversarial Holdout", report["acceptance_gates"]["adversarial_holdout_non_regression"]), + ): + winner_health = gate.get("winner_family_health") or {} + current_health = gate.get("current_family_health") or {} + baseline_health = gate.get("baseline_family_health") or {} + if not winner_health and not current_health and not baseline_health: + continue + weakest = winner_health.get("weakest_family") or {} + weakest_label = ( + f"{weakest.get('family')} ({weakest.get('errors')} errors)" + if weakest.get("family") + else "-" + ) + lines.append( + f"| {gate_name} | {winner_health.get('clean_family_count', '-')}/{winner_health.get('family_count', '-')} | {weakest_label} | {current_health.get('clean_family_count', '-')}/{current_health.get('family_count', '-')} | {baseline_health.get('clean_family_count', '-')}/{baseline_health.get('family_count', '-')} |" + ) + lines.extend( [ "", @@ -390,12 +614,15 @@ def render_markdown(report: dict, title: str) -> str: def main() -> None: - parser = argparse.ArgumentParser(description="Generate and score description candidates on dev and holdout suites.") + parser = argparse.ArgumentParser( + description="Generate and score description candidates on dev, holdout, blind, and adversarial suites." + ) parser.add_argument("--description-file", required=True) parser.add_argument("--baseline-description-file") parser.add_argument("--dev-cases", required=True) parser.add_argument("--holdout-cases") parser.add_argument("--blind-holdout-cases") + parser.add_argument("--adversarial-cases") parser.add_argument("--semantic-config", required=True) parser.add_argument("--output-json") parser.add_argument("--output-md") @@ -407,9 +634,18 @@ def main() -> None: dev_cases = load_json(Path(args.dev_cases)) holdout_cases = load_json(Path(args.holdout_cases)) if args.holdout_cases else None blind_holdout_cases = load_json(Path(args.blind_holdout_cases)) if args.blind_holdout_cases else None + adversarial_cases = load_json(Path(args.adversarial_cases)) if args.adversarial_cases else None config = load_semantic_config(Path(args.semantic_config)) - report = optimize(current_description, dev_cases, holdout_cases, config, baseline_description, blind_holdout_cases) + report = optimize( + current_description, + dev_cases, + holdout_cases, + config, + baseline_description, + blind_holdout_cases, + adversarial_cases, + ) rendered = json.dumps(report, ensure_ascii=False, indent=2) if args.output_json: Path(args.output_json).write_text(rendered + "\n", encoding="utf-8") diff --git a/scripts/render_description_drift_history.py b/scripts/render_description_drift_history.py index 42210b6..adab4b7 100644 --- a/scripts/render_description_drift_history.py +++ b/scripts/render_description_drift_history.py @@ -15,13 +15,23 @@ def render_markdown(snapshots: list[dict]) -> str: lines = [ "# Description Drift History", "", - "| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Drift Note |", - "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | --- |", + "| Date | Label | Target | Winner | Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Drift Note |", + "| --- | --- | --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", ] previous_by_target: dict[str, dict] = {} + def display(value: object) -> str: return "-" if value is None else str(value) + + def family_display(target: dict, gate: str) -> str: + family = (target.get("family_health", {}) or {}).get(gate) or {} + if not family: + return "-" + weakest = family.get("weakest_family") or {} + weakest_label = weakest.get("family") or "-" + return f"{family.get('clean_family_count', 0)}/{family.get('family_count', 0)} clean; weakest={weakest_label}" + for snapshot in snapshots: for target in snapshot.get("targets", []): previous = previous_by_target.get(target["name"]) @@ -35,13 +45,35 @@ def render_markdown(snapshots: list[dict]) -> str: blind_note = "blind gate unchanged" else: blind_note = f"blind error delta {blind_now - blind_prev:+d}" - drift_note = f"{token_note}; {blind_note}" + adv_prev = previous.get("winner_adversarial_holdout_total_errors") + adv_now = target.get("winner_adversarial_holdout_total_errors") + if adv_prev is None or adv_now is None: + adv_note = "adversarial gate unchanged" + else: + adv_note = f"adversarial error delta {adv_now - adv_prev:+d}" + drift_note = f"{token_note}; {blind_note}; {adv_note}" if not drift_note: drift_note = "initial snapshot" + adversarial_calibration = (target.get("calibration", {}) or {}).get("adversarial_holdout") or {} lines.append( - f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {display(target.get('winner_blind_holdout_fp'))} | {display(target.get('winner_blind_holdout_fn'))} | {drift_note} |" + f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {display(target.get('winner_blind_holdout_fp'))} | {display(target.get('winner_blind_holdout_fn'))} | {display(target.get('winner_adversarial_holdout_fp'))} | {display(target.get('winner_adversarial_holdout_fn'))} | {display(adversarial_calibration.get('score_gap'))} | {display(adversarial_calibration.get('risk_band'))} | {drift_note} |" ) previous_by_target[target["name"]] = target + + lines.extend( + [ + "", + "## Family Coverage", + "", + "| Date | Label | Target | Blind Families | Adversarial Families |", + "| --- | --- | --- | --- | --- |", + ] + ) + for snapshot in snapshots: + for target in snapshot.get("targets", []): + lines.append( + f"| {snapshot['date']} | {snapshot['label']} | `{target['name']}` | {family_display(target, 'blind_holdout')} | {family_display(target, 'adversarial_holdout')} |" + ) return "\n".join(lines) + "\n" diff --git a/scripts/run_description_optimization_suite.py b/scripts/run_description_optimization_suite.py index 0bebff1..1dcdada 100644 --- a/scripts/run_description_optimization_suite.py +++ b/scripts/run_description_optimization_suite.py @@ -19,6 +19,7 @@ TARGETS = [ "dev_cases": ROOT / "evals" / "dev" / "trigger_cases.json", "holdout_cases": ROOT / "evals" / "holdout" / "trigger_cases.json", "blind_holdout_cases": ROOT / "evals" / "blind_holdout" / "trigger_cases.json", + "adversarial_cases": ROOT / "evals" / "adversarial" / "trigger_cases.json", "semantic_config": ROOT / "evals" / "semantic_config.json", "output_json": ROOT / "reports" / "description_optimization.json", "output_md": ROOT / "reports" / "description_optimization.md", @@ -31,6 +32,7 @@ TARGETS = [ "dev_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "dev" / "trigger_cases.json", "holdout_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "holdout" / "trigger_cases.json", "blind_holdout_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "blind_holdout" / "trigger_cases.json", + "adversarial_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "adversarial" / "trigger_cases.json", "semantic_config": ROOT / "examples" / "team-frontend-review" / "optimization" / "semantic_config.json", "output_json": ROOT / "examples" / "team-frontend-review" / "optimization" / "reports" / "description_optimization.json", "output_md": ROOT / "examples" / "team-frontend-review" / "optimization" / "reports" / "description_optimization.md", @@ -43,6 +45,7 @@ TARGETS = [ "dev_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "dev" / "trigger_cases.json", "holdout_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "holdout" / "trigger_cases.json", "blind_holdout_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "blind_holdout" / "trigger_cases.json", + "adversarial_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "adversarial" / "trigger_cases.json", "semantic_config": ROOT / "examples" / "governed-incident-command" / "optimization" / "semantic_config.json", "output_json": ROOT / "examples" / "governed-incident-command" / "optimization" / "reports" / "description_optimization.json", "output_md": ROOT / "examples" / "governed-incident-command" / "optimization" / "reports" / "description_optimization.md", @@ -76,6 +79,20 @@ def target_error_total(target: dict, prefix: str) -> int | None: return fp + fn +def calibration_gap(target: dict, gate: str) -> float | None: + calibration = target.get("calibration", {}).get(gate) or {} + return calibration.get("score_gap") + + +def family_gate_note(target: dict, gate: str) -> str: + family = target.get("family_health", {}).get(gate) or {} + if not family: + return "n/a" + weakest = family.get("weakest_family") or {} + weakest_label = weakest.get("family") or "-" + return f"{family.get('clean_family_count', 0)}/{family.get('family_count', 0)} clean; weakest={weakest_label}" + + def drift_note_for_target(target: dict, previous: dict | None) -> str: if not previous: return "initial description optimization snapshot" @@ -98,6 +115,17 @@ def drift_note_for_target(target: dict, previous: dict | None) -> str: else: notes.append(f"blind error delta {delta:+d}") + previous_adv = previous.get("winner_adversarial_holdout_total_errors") + current_adv = target.get("winner_adversarial_holdout_total_errors") + if previous_adv is None and current_adv is not None: + notes.append(f"adversarial gate added with {current_adv} errors") + elif previous_adv is not None and current_adv is not None: + delta = current_adv - previous_adv + if delta == 0: + notes.append(f"adversarial stable at {current_adv}") + else: + notes.append(f"adversarial error delta {delta:+d}") + previous_holdout = target_error_total(previous, "winner_holdout") current_holdout = target_error_total(target, "winner_holdout") if previous_holdout is not None and current_holdout is not None: @@ -107,6 +135,22 @@ def drift_note_for_target(target: dict, previous: dict | None) -> str: else: notes.append(f"holdout error delta {delta:+d}") + previous_gap = calibration_gap(previous, "adversarial_holdout") + current_gap = calibration_gap(target, "adversarial_holdout") + if previous_gap is None and current_gap is not None: + notes.append(f"adversarial calibration {current_gap:+.3f}") + elif previous_gap is not None and current_gap is not None: + delta = current_gap - previous_gap + if abs(delta) < 0.001: + notes.append(f"adversarial calibration stable at {current_gap:+.3f}") + else: + notes.append(f"adversarial calibration delta {delta:+.3f}") + + previous_risk = (previous.get("calibration", {}).get("adversarial_holdout") or {}).get("risk_band") + current_risk = (target.get("calibration", {}).get("adversarial_holdout") or {}).get("risk_band") + if previous_risk != current_risk and current_risk: + notes.append(f"risk {previous_risk or 'n/a'} -> {current_risk}") + return "; ".join(notes) @@ -130,8 +174,8 @@ def build_history_snapshot(summary: dict, args: argparse.Namespace) -> dict: "label": args.snapshot_label, "targets": targets, "notes": [ - "added blind holdout acceptance gates to description optimization", - "published description drift history report", + "recorded family-level blind and adversarial routing evidence", + "published calibration and drift history for description optimization", ], } @@ -140,8 +184,8 @@ def main() -> None: parser = argparse.ArgumentParser(description="Run description optimization across root and example skills.") parser.add_argument("--history-snapshot-output") parser.add_argument("--snapshot-date") - parser.add_argument("--snapshot-id", default="blind-holdout-and-drift-history") - parser.add_argument("--snapshot-label", default="Blind Holdout And Drift History") + parser.add_argument("--snapshot-id", default="adversarial-calibration-and-family-drift") + parser.add_argument("--snapshot-label", default="Adversarial Calibration And Family Drift") parser.add_argument("--snapshot-commit", default="local-snapshot") args = parser.parse_args() @@ -152,9 +196,18 @@ def main() -> None: dev_cases = load_json(target["dev_cases"]) holdout_cases = load_json(target["holdout_cases"]) blind_holdout_cases = load_json(target["blind_holdout_cases"]) + adversarial_cases = load_json(target["adversarial_cases"]) config = load_semantic_config(target["semantic_config"]) - report = optimize(current_description, dev_cases, holdout_cases, config, baseline_description, blind_holdout_cases) + report = optimize( + current_description, + dev_cases, + holdout_cases, + config, + baseline_description, + blind_holdout_cases, + adversarial_cases, + ) target["output_json"].parent.mkdir(parents=True, exist_ok=True) target["output_json"].write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") target["output_md"].write_text(render_markdown(report, target["title"]), encoding="utf-8") @@ -165,12 +218,23 @@ def main() -> None: blind_winner_fp, blind_winner_fn = report_errors(report["acceptance_gates"]["blind_holdout_non_regression"]["winner"]) blind_current_fp, blind_current_fn = report_errors(report["acceptance_gates"]["blind_holdout_non_regression"]["current"]) blind_baseline_fp, blind_baseline_fn = report_errors(report["acceptance_gates"]["blind_holdout_non_regression"]["baseline"]) + adversarial_winner_fp, adversarial_winner_fn = report_errors( + report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner"] + ) + adversarial_current_fp, adversarial_current_fn = report_errors( + report["acceptance_gates"]["adversarial_holdout_non_regression"]["current"] + ) + adversarial_baseline_fp, adversarial_baseline_fn = report_errors( + report["acceptance_gates"]["adversarial_holdout_non_regression"]["baseline"] + ) target_ok = ( (winner_fp, winner_fn) <= (current_fp, current_fn) and (winner_fp, winner_fn) <= (baseline_fp, baseline_fn) and (blind_winner_fp, blind_winner_fn) <= (blind_current_fp, blind_current_fn) and (blind_winner_fp, blind_winner_fn) <= (blind_baseline_fp, blind_baseline_fn) + and (adversarial_winner_fp, adversarial_winner_fn) <= (adversarial_current_fp, adversarial_current_fn) + and (adversarial_winner_fp, adversarial_winner_fn) <= (adversarial_baseline_fp, adversarial_baseline_fn) ) summary["targets"].append( { @@ -192,7 +256,24 @@ def main() -> None: "baseline_blind_holdout_fp": blind_baseline_fp, "baseline_blind_holdout_fn": blind_baseline_fn, "winner_blind_holdout_total_errors": blind_winner_fp + blind_winner_fn, - "drift_note": "blind holdout gate active", + "winner_adversarial_holdout_fp": adversarial_winner_fp, + "winner_adversarial_holdout_fn": adversarial_winner_fn, + "current_adversarial_holdout_fp": adversarial_current_fp, + "current_adversarial_holdout_fn": adversarial_current_fn, + "baseline_adversarial_holdout_fp": adversarial_baseline_fp, + "baseline_adversarial_holdout_fn": adversarial_baseline_fn, + "winner_adversarial_holdout_total_errors": adversarial_winner_fp + adversarial_winner_fn, + "calibration": { + "holdout": report["acceptance_gates"]["holdout_non_regression"]["winner_calibration"], + "blind_holdout": report["acceptance_gates"]["blind_holdout_non_regression"]["winner_calibration"], + "adversarial_holdout": report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"], + }, + "family_health": { + "holdout": report["acceptance_gates"]["holdout_non_regression"]["winner_family_health"], + "blind_holdout": report["acceptance_gates"]["blind_holdout_non_regression"]["winner_family_health"], + "adversarial_holdout": report["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_family_health"], + }, + "drift_note": "blind, adversarial, and calibration gates active", "ok": target_ok, } ) @@ -204,12 +285,25 @@ def main() -> None: lines = [ "# Description Optimization Suite", "", - "| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Current Blind FN | Baseline Blind FN | Status |", - "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- |", + "| Target | Winner | Winner Tokens | Holdout FP | Holdout FN | Blind FP | Blind FN | Adv FP | Adv FN | Adv Gap | Adv Risk | Status |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | --- | --- |", ] for target in summary["targets"]: lines.append( - f"| `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {target['winner_blind_holdout_fp']} | {target['winner_blind_holdout_fn']} | {target['current_blind_holdout_fn']} | {target['baseline_blind_holdout_fn']} | {'ok' if target['ok'] else 'fail'} |" + f"| `{target['name']}` | `{target['winner_label']}` | {target['winner_tokens']} | {target['winner_holdout_fp']} | {target['winner_holdout_fn']} | {target['winner_blind_holdout_fp']} | {target['winner_blind_holdout_fn']} | {target['winner_adversarial_holdout_fp']} | {target['winner_adversarial_holdout_fn']} | {(target['calibration']['adversarial_holdout'] or {}).get('score_gap', '-')} | {(target['calibration']['adversarial_holdout'] or {}).get('risk_band', '-')} | {'ok' if target['ok'] else 'fail'} |" + ) + lines.extend( + [ + "", + "## Family Coverage", + "", + "| Target | Blind Families | Adversarial Families |", + "| --- | --- | --- |", + ] + ) + for target in summary["targets"]: + lines.append( + f"| `{target['name']}` | {family_gate_note(target, 'blind_holdout')} | {family_gate_note(target, 'adversarial_holdout')} |" ) (ROOT / "reports" / "description_optimization_suite.md").write_text("\n".join(lines) + "\n", encoding="utf-8") if args.history_snapshot_output: diff --git a/tests/verify_description_optimization.py b/tests/verify_description_optimization.py index 256ddbb..070326b 100644 --- a/tests/verify_description_optimization.py +++ b/tests/verify_description_optimization.py @@ -17,6 +17,7 @@ CASES = [ "dev_cases": ROOT / "evals" / "dev" / "trigger_cases.json", "holdout_cases": ROOT / "evals" / "holdout" / "trigger_cases.json", "blind_holdout_cases": ROOT / "evals" / "blind_holdout" / "trigger_cases.json", + "adversarial_cases": ROOT / "evals" / "adversarial" / "trigger_cases.json", }, { "name": "team_frontend_review", @@ -26,6 +27,7 @@ CASES = [ "dev_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "dev" / "trigger_cases.json", "holdout_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "holdout" / "trigger_cases.json", "blind_holdout_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "blind_holdout" / "trigger_cases.json", + "adversarial_cases": ROOT / "examples" / "team-frontend-review" / "optimization" / "adversarial" / "trigger_cases.json", }, { "name": "governed_incident_command", @@ -35,6 +37,7 @@ CASES = [ "dev_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "dev" / "trigger_cases.json", "holdout_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "holdout" / "trigger_cases.json", "blind_holdout_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "blind_holdout" / "trigger_cases.json", + "adversarial_cases": ROOT / "examples" / "governed-incident-command" / "optimization" / "adversarial" / "trigger_cases.json", }, ] @@ -60,6 +63,8 @@ def run_case(case: dict) -> dict: str(case["holdout_cases"]), "--blind-holdout-cases", str(case["blind_holdout_cases"]), + "--adversarial-cases", + str(case["adversarial_cases"]), ], cwd=ROOT, capture_output=True, @@ -81,6 +86,17 @@ def run_case(case: dict) -> dict: passed = passed and blind_winner is not None and total_errors(blind_winner) <= total_errors(blind_current) if blind_baseline: passed = passed and total_errors(blind_winner) <= total_errors(blind_baseline) + adversarial_winner = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["winner"] + adversarial_current = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["current"] + adversarial_baseline = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["baseline"] + passed = passed and adversarial_winner is not None and total_errors(adversarial_winner) <= total_errors(adversarial_current) + if adversarial_baseline: + passed = passed and total_errors(adversarial_winner) <= total_errors(adversarial_baseline) + winner_adversarial_calibration = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_calibration"] + winner_adversarial_family = payload["acceptance_gates"]["adversarial_holdout_non_regression"]["winner_family_health"] + passed = passed and winner_adversarial_calibration is not None + passed = passed and winner_adversarial_family is not None + passed = passed and winner_adversarial_family.get("family_count", 0) >= 2 passed = passed and len(payload.get("candidates", [])) >= 3 return { "name": case["name"], @@ -92,6 +108,8 @@ def run_case(case: dict) -> dict: "current_tokens": current["estimated_tokens"], "baseline_tokens": baseline["estimated_tokens"] if baseline else None, "winner_blind_holdout_total_errors": total_errors(blind_winner) if blind_winner else None, + "winner_adversarial_holdout_total_errors": total_errors(adversarial_winner) if adversarial_winner else None, + "winner_adversarial_risk_band": winner_adversarial_calibration.get("risk_band") if winner_adversarial_calibration else None, }